2024-12-10 11:03:41,871 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-12-10 11:03:41,885 main DEBUG Took 0.011390 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 11:03:41,885 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 11:03:41,886 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 11:03:41,886 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 11:03:41,888 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,897 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 11:03:41,912 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,914 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,915 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,916 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,917 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,918 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,919 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,920 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,920 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,921 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,921 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,921 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,922 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,922 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,922 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,923 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,923 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,923 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,924 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,924 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 11:03:41,925 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,925 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 11:03:41,926 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 11:03:41,928 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 11:03:41,930 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 11:03:41,930 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 11:03:41,931 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 11:03:41,931 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 11:03:41,943 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 11:03:41,946 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 11:03:41,948 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 11:03:41,948 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 11:03:41,949 main DEBUG createAppenders(={Console}) 2024-12-10 11:03:41,950 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-12-10 11:03:41,950 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-12-10 11:03:41,951 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-12-10 11:03:41,951 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 11:03:41,952 main DEBUG OutputStream closed 2024-12-10 11:03:41,952 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 11:03:41,953 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 11:03:41,953 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-12-10 11:03:42,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 11:03:42,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 11:03:42,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 11:03:42,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 11:03:42,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 11:03:42,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 11:03:42,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 11:03:42,040 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 11:03:42,040 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 11:03:42,041 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 11:03:42,041 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 11:03:42,042 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 11:03:42,042 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 11:03:42,043 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 11:03:42,043 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 11:03:42,043 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 11:03:42,044 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 11:03:42,044 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 11:03:42,047 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 11:03:42,047 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-12-10 11:03:42,047 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 11:03:42,048 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-12-10T11:03:42,063 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-12-10 11:03:42,065 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 11:03:42,066 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T11:03:42,113 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=195, ProcessCount=11, AvailableMemoryMB=5391 2024-12-10T11:03:42,364 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9 2024-12-10T11:03:42,930 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T11:03:43,050 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T11:03:43,106 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-12-10T11:03:43,164 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623152 2024-12-10T11:03:43,236 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623152 2024-12-10T11:03:43,321 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,327 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-12-10T11:03:43,327 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-12-10T11:03:43,365 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0dbc67421fdaf4442fb699251a0dc6f4, NAME => 'testLockupAroundBadAssignSync,,1733828623341.0dbc67421fdaf4442fb699251a0dc6f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9 2024-12-10T11:03:43,412 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T11:03:43,416 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1733828623341.0dbc67421fdaf4442fb699251a0dc6f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T11:03:43,501 INFO [StoreOpener-0dbc67421fdaf4442fb699251a0dc6f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region 0dbc67421fdaf4442fb699251a0dc6f4 2024-12-10T11:03:43,535 INFO [StoreOpener-0dbc67421fdaf4442fb699251a0dc6f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dbc67421fdaf4442fb699251a0dc6f4 columnFamilyName MyCF 2024-12-10T11:03:43,542 DEBUG [StoreOpener-0dbc67421fdaf4442fb699251a0dc6f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T11:03:43,547 INFO [StoreOpener-0dbc67421fdaf4442fb699251a0dc6f4-1 {}] regionserver.HStore(327): Store=0dbc67421fdaf4442fb699251a0dc6f4/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T11:03:43,551 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0dbc67421fdaf4442fb699251a0dc6f4 2024-12-10T11:03:43,559 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/data/default/testLockupAroundBadAssignSync/0dbc67421fdaf4442fb699251a0dc6f4 2024-12-10T11:03:43,559 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/data/default/testLockupAroundBadAssignSync/0dbc67421fdaf4442fb699251a0dc6f4 2024-12-10T11:03:43,563 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0dbc67421fdaf4442fb699251a0dc6f4 2024-12-10T11:03:43,564 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0dbc67421fdaf4442fb699251a0dc6f4 2024-12-10T11:03:43,572 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0dbc67421fdaf4442fb699251a0dc6f4 2024-12-10T11:03:43,584 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/data/default/testLockupAroundBadAssignSync/0dbc67421fdaf4442fb699251a0dc6f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T11:03:43,586 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0dbc67421fdaf4442fb699251a0dc6f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72145259, jitterRate=0.07504813373088837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T11:03:43,594 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T11:03:43,611 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0dbc67421fdaf4442fb699251a0dc6f4: Writing region info on filesystem at 1733828623463Initializing all the Stores at 1733828623464 (+1 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1733828623465 (+1 ms)Cleaning up temporary data from old regions at 1733828623564 (+99 ms)Region opened successfully at 1733828623608 (+44 ms) 2024-12-10T11:03:43,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623152) roll requested 2024-12-10T11:03:43,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623666 2024-12-10T11:03:43,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623152 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623666 2024-12-10T11:03:43,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623666) roll requested 2024-12-10T11:03:43,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623679 2024-12-10T11:03:43,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623679 2024-12-10T11:03:43,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623679) roll requested 2024-12-10T11:03:43,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623693 2024-12-10T11:03:43,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623693 2024-12-10T11:03:43,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623693) roll requested 2024-12-10T11:03:43,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623703 2024-12-10T11:03:43,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623703 2024-12-10T11:03:43,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623703) roll requested 2024-12-10T11:03:43,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623713 2024-12-10T11:03:43,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623713 2024-12-10T11:03:43,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623713) roll requested 2024-12-10T11:03:43,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623724 2024-12-10T11:03:43,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623724 2024-12-10T11:03:43,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623724) roll requested 2024-12-10T11:03:43,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623735 2024-12-10T11:03:43,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623735 2024-12-10T11:03:43,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623735) roll requested 2024-12-10T11:03:43,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623745 2024-12-10T11:03:43,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623745 2024-12-10T11:03:43,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623745) roll requested 2024-12-10T11:03:43,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623756 2024-12-10T11:03:43,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623756 2024-12-10T11:03:43,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623756) roll requested 2024-12-10T11:03:43,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623766 2024-12-10T11:03:43,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623766 2024-12-10T11:03:43,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623766) roll requested 2024-12-10T11:03:43,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623777 2024-12-10T11:03:43,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623777 2024-12-10T11:03:43,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623777) roll requested 2024-12-10T11:03:43,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623787 2024-12-10T11:03:43,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623787 2024-12-10T11:03:43,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623787) roll requested 2024-12-10T11:03:43,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623797 2024-12-10T11:03:43,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623797 2024-12-10T11:03:43,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623797) roll requested 2024-12-10T11:03:43,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623807 2024-12-10T11:03:43,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623807 2024-12-10T11:03:43,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623807) roll requested 2024-12-10T11:03:43,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623816 2024-12-10T11:03:43,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623816 2024-12-10T11:03:43,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623816) roll requested 2024-12-10T11:03:43,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623827 2024-12-10T11:03:43,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623827 2024-12-10T11:03:43,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623827) roll requested 2024-12-10T11:03:43,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623836 2024-12-10T11:03:43,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623836 2024-12-10T11:03:43,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623836) roll requested 2024-12-10T11:03:43,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623846 2024-12-10T11:03:43,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623846 2024-12-10T11:03:43,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623846) roll requested 2024-12-10T11:03:43,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623856 2024-12-10T11:03:43,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623856 2024-12-10T11:03:43,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623856) roll requested 2024-12-10T11:03:43,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623867 2024-12-10T11:03:43,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623867 2024-12-10T11:03:43,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623867) roll requested 2024-12-10T11:03:43,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623877 2024-12-10T11:03:43,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623877 2024-12-10T11:03:43,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623877) roll requested 2024-12-10T11:03:43,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623892 2024-12-10T11:03:43,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623892 2024-12-10T11:03:43,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623892) roll requested 2024-12-10T11:03:43,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623902 2024-12-10T11:03:43,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623902 2024-12-10T11:03:43,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623902) roll requested 2024-12-10T11:03:43,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623915 2024-12-10T11:03:43,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623915 2024-12-10T11:03:43,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623915) roll requested 2024-12-10T11:03:43,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623925 2024-12-10T11:03:43,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623925 2024-12-10T11:03:43,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623925) roll requested 2024-12-10T11:03:43,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623934 2024-12-10T11:03:43,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623934 2024-12-10T11:03:43,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623934) roll requested 2024-12-10T11:03:43,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623943 2024-12-10T11:03:43,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623943 2024-12-10T11:03:43,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623943) roll requested 2024-12-10T11:03:43,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623952 2024-12-10T11:03:43,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623952 2024-12-10T11:03:43,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623952) roll requested 2024-12-10T11:03:43,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623960 2024-12-10T11:03:43,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623960 2024-12-10T11:03:43,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623960) roll requested 2024-12-10T11:03:43,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623972 2024-12-10T11:03:43,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623972 2024-12-10T11:03:43,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623972) roll requested 2024-12-10T11:03:43,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623981 2024-12-10T11:03:43,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623981 2024-12-10T11:03:43,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623981) roll requested 2024-12-10T11:03:43,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623989 2024-12-10T11:03:43,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:43,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623989 2024-12-10T11:03:43,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:43,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:43,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623989) roll requested 2024-12-10T11:03:43,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828623999 2024-12-10T11:03:44,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623999 2024-12-10T11:03:44,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828623999) roll requested 2024-12-10T11:03:44,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624007 2024-12-10T11:03:44,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828623999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624007 2024-12-10T11:03:44,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624007) roll requested 2024-12-10T11:03:44,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624015 2024-12-10T11:03:44,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624015 2024-12-10T11:03:44,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624015) roll requested 2024-12-10T11:03:44,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624024 2024-12-10T11:03:44,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624024 2024-12-10T11:03:44,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624024) roll requested 2024-12-10T11:03:44,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624037 2024-12-10T11:03:44,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624037 2024-12-10T11:03:44,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624037) roll requested 2024-12-10T11:03:44,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624045 2024-12-10T11:03:44,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624045 2024-12-10T11:03:44,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624045) roll requested 2024-12-10T11:03:44,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624055 2024-12-10T11:03:44,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624055 2024-12-10T11:03:44,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624055) roll requested 2024-12-10T11:03:44,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624064 2024-12-10T11:03:44,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624064 2024-12-10T11:03:44,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624064) roll requested 2024-12-10T11:03:44,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624072 2024-12-10T11:03:44,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624072 2024-12-10T11:03:44,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624072) roll requested 2024-12-10T11:03:44,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624080 2024-12-10T11:03:44,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624080 2024-12-10T11:03:44,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624080) roll requested 2024-12-10T11:03:44,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624088 2024-12-10T11:03:44,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624088 2024-12-10T11:03:44,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624088) roll requested 2024-12-10T11:03:44,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624098 2024-12-10T11:03:44,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624098 2024-12-10T11:03:44,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624098) roll requested 2024-12-10T11:03:44,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624106 2024-12-10T11:03:44,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624106 2024-12-10T11:03:44,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624106) roll requested 2024-12-10T11:03:44,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624115 2024-12-10T11:03:44,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624115 2024-12-10T11:03:44,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624115) roll requested 2024-12-10T11:03:44,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624122 2024-12-10T11:03:44,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624122 2024-12-10T11:03:44,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624122) roll requested 2024-12-10T11:03:44,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624131 2024-12-10T11:03:44,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624131 2024-12-10T11:03:44,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624131) roll requested 2024-12-10T11:03:44,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624139 2024-12-10T11:03:44,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624139 2024-12-10T11:03:44,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624139) roll requested 2024-12-10T11:03:44,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624148 2024-12-10T11:03:44,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624148 2024-12-10T11:03:44,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624148) roll requested 2024-12-10T11:03:44,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624157 2024-12-10T11:03:44,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624157 2024-12-10T11:03:44,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624157) roll requested 2024-12-10T11:03:44,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624164 2024-12-10T11:03:44,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624164 2024-12-10T11:03:44,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624164) roll requested 2024-12-10T11:03:44,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624172 2024-12-10T11:03:44,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624172 2024-12-10T11:03:44,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624172) roll requested 2024-12-10T11:03:44,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624180 2024-12-10T11:03:44,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624180 2024-12-10T11:03:44,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624180) roll requested 2024-12-10T11:03:44,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624189 2024-12-10T11:03:44,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624189 2024-12-10T11:03:44,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624189) roll requested 2024-12-10T11:03:44,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624198 2024-12-10T11:03:44,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624198 2024-12-10T11:03:44,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624198) roll requested 2024-12-10T11:03:44,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624205 2024-12-10T11:03:44,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624205 2024-12-10T11:03:44,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624205) roll requested 2024-12-10T11:03:44,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624218 2024-12-10T11:03:44,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624218 2024-12-10T11:03:44,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624218) roll requested 2024-12-10T11:03:44,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624226 2024-12-10T11:03:44,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624226 2024-12-10T11:03:44,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624226) roll requested 2024-12-10T11:03:44,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624234 2024-12-10T11:03:44,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624234 2024-12-10T11:03:44,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624234) roll requested 2024-12-10T11:03:44,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624242 2024-12-10T11:03:44,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624242 2024-12-10T11:03:44,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624242) roll requested 2024-12-10T11:03:44,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624250 2024-12-10T11:03:44,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624250 2024-12-10T11:03:44,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624250) roll requested 2024-12-10T11:03:44,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624261 2024-12-10T11:03:44,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624261 2024-12-10T11:03:44,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624261) roll requested 2024-12-10T11:03:44,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624270 2024-12-10T11:03:44,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624270 2024-12-10T11:03:44,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624261 is not closed yet, will try archiving it next time 2024-12-10T11:03:44,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624270) roll requested 2024-12-10T11:03:44,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624278 2024-12-10T11:03:44,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624278 2024-12-10T11:03:44,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624278) roll requested 2024-12-10T11:03:44,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624285 2024-12-10T11:03:44,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624285 2024-12-10T11:03:44,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624285) roll requested 2024-12-10T11:03:44,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624293 2024-12-10T11:03:44,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624293 2024-12-10T11:03:44,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624293) roll requested 2024-12-10T11:03:44,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624300 2024-12-10T11:03:44,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624300 2024-12-10T11:03:44,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624300) roll requested 2024-12-10T11:03:44,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624309 2024-12-10T11:03:44,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624309 2024-12-10T11:03:44,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624309) roll requested 2024-12-10T11:03:44,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624316 2024-12-10T11:03:44,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624316 2024-12-10T11:03:44,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624316) roll requested 2024-12-10T11:03:44,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624323 2024-12-10T11:03:44,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624323 2024-12-10T11:03:44,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624323) roll requested 2024-12-10T11:03:44,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624331 2024-12-10T11:03:44,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624331 2024-12-10T11:03:44,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624331) roll requested 2024-12-10T11:03:44,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624339 2024-12-10T11:03:44,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624339 2024-12-10T11:03:44,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624339) roll requested 2024-12-10T11:03:44,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624346 2024-12-10T11:03:44,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624346 2024-12-10T11:03:44,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624346) roll requested 2024-12-10T11:03:44,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624354 2024-12-10T11:03:44,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624354 2024-12-10T11:03:44,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624354) roll requested 2024-12-10T11:03:44,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624361 2024-12-10T11:03:44,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624361 2024-12-10T11:03:44,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624361) roll requested 2024-12-10T11:03:44,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624369 2024-12-10T11:03:44,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624369 2024-12-10T11:03:44,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624369) roll requested 2024-12-10T11:03:44,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624376 2024-12-10T11:03:44,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624376 2024-12-10T11:03:44,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624376) roll requested 2024-12-10T11:03:44,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624384 2024-12-10T11:03:44,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624384 2024-12-10T11:03:44,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624384) roll requested 2024-12-10T11:03:44,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624392 2024-12-10T11:03:44,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624392 2024-12-10T11:03:44,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624392) roll requested 2024-12-10T11:03:44,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624399 2024-12-10T11:03:44,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624399 2024-12-10T11:03:44,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624399) roll requested 2024-12-10T11:03:44,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624407 2024-12-10T11:03:44,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624407 2024-12-10T11:03:44,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624407) roll requested 2024-12-10T11:03:44,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624414 2024-12-10T11:03:44,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624414 2024-12-10T11:03:44,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624414) roll requested 2024-12-10T11:03:44,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624421 2024-12-10T11:03:44,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624421 2024-12-10T11:03:44,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624421) roll requested 2024-12-10T11:03:44,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624430 2024-12-10T11:03:44,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624430 2024-12-10T11:03:44,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624430) roll requested 2024-12-10T11:03:44,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624437 2024-12-10T11:03:44,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624437 2024-12-10T11:03:44,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624437) roll requested 2024-12-10T11:03:44,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624448 2024-12-10T11:03:44,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624448 2024-12-10T11:03:44,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624448) roll requested 2024-12-10T11:03:44,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624455 2024-12-10T11:03:44,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624455 2024-12-10T11:03:44,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624455) roll requested 2024-12-10T11:03:44,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624466 2024-12-10T11:03:44,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624466 2024-12-10T11:03:44,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624466) roll requested 2024-12-10T11:03:44,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624477 2024-12-10T11:03:44,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624477 2024-12-10T11:03:44,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624477) roll requested 2024-12-10T11:03:44,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624487 2024-12-10T11:03:44,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624487 2024-12-10T11:03:44,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624487) roll requested 2024-12-10T11:03:44,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624494 2024-12-10T11:03:44,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624494 2024-12-10T11:03:44,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624494) roll requested 2024-12-10T11:03:44,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624501 2024-12-10T11:03:44,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624501 2024-12-10T11:03:44,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624501) roll requested 2024-12-10T11:03:44,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624508 2024-12-10T11:03:44,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624508 2024-12-10T11:03:44,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624508) roll requested 2024-12-10T11:03:44,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624517 2024-12-10T11:03:44,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624517 2024-12-10T11:03:44,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624517) roll requested 2024-12-10T11:03:44,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624525 2024-12-10T11:03:44,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624525 2024-12-10T11:03:44,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624525) roll requested 2024-12-10T11:03:44,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624538 2024-12-10T11:03:44,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624538 2024-12-10T11:03:44,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624538) roll requested 2024-12-10T11:03:44,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624545 2024-12-10T11:03:44,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624545 2024-12-10T11:03:44,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624545) roll requested 2024-12-10T11:03:44,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624552 2024-12-10T11:03:44,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624552 2024-12-10T11:03:44,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624552) roll requested 2024-12-10T11:03:44,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624560 2024-12-10T11:03:44,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624560 2024-12-10T11:03:44,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624560) roll requested 2024-12-10T11:03:44,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624567 2024-12-10T11:03:44,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624567 2024-12-10T11:03:44,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624567) roll requested 2024-12-10T11:03:44,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624575 2024-12-10T11:03:44,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624575 2024-12-10T11:03:44,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624575) roll requested 2024-12-10T11:03:44,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624582 2024-12-10T11:03:44,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624582 2024-12-10T11:03:44,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624582) roll requested 2024-12-10T11:03:44,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624589 2024-12-10T11:03:44,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624589 2024-12-10T11:03:44,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624589) roll requested 2024-12-10T11:03:44,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624596 2024-12-10T11:03:44,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624596 2024-12-10T11:03:44,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624596) roll requested 2024-12-10T11:03:44,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624603 2024-12-10T11:03:44,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624603 2024-12-10T11:03:44,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624603) roll requested 2024-12-10T11:03:44,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624610 2024-12-10T11:03:44,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624610 2024-12-10T11:03:44,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624610) roll requested 2024-12-10T11:03:44,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624617 2024-12-10T11:03:44,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624617 2024-12-10T11:03:44,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624617) roll requested 2024-12-10T11:03:44,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624624 2024-12-10T11:03:44,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624624 2024-12-10T11:03:44,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624624) roll requested 2024-12-10T11:03:44,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624631 2024-12-10T11:03:44,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624631 2024-12-10T11:03:44,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624631) roll requested 2024-12-10T11:03:44,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624639 2024-12-10T11:03:44,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624639 2024-12-10T11:03:44,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624639) roll requested 2024-12-10T11:03:44,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624646 2024-12-10T11:03:44,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624646 2024-12-10T11:03:44,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624646) roll requested 2024-12-10T11:03:44,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624654 2024-12-10T11:03:44,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624654 2024-12-10T11:03:44,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624646 is not closed yet, will try archiving it next time 2024-12-10T11:03:44,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624654) roll requested 2024-12-10T11:03:44,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624661 2024-12-10T11:03:44,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624661 2024-12-10T11:03:44,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624661) roll requested 2024-12-10T11:03:44,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624669 2024-12-10T11:03:44,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624669 2024-12-10T11:03:44,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624669) roll requested 2024-12-10T11:03:44,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624676 2024-12-10T11:03:44,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624676 2024-12-10T11:03:44,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624676) roll requested 2024-12-10T11:03:44,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624687 2024-12-10T11:03:44,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624687 2024-12-10T11:03:44,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624687) roll requested 2024-12-10T11:03:44,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624696 2024-12-10T11:03:44,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624696 2024-12-10T11:03:44,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624696) roll requested 2024-12-10T11:03:44,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624704 2024-12-10T11:03:44,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624704 2024-12-10T11:03:44,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624704) roll requested 2024-12-10T11:03:44,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624712 2024-12-10T11:03:44,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624712 2024-12-10T11:03:44,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624712) roll requested 2024-12-10T11:03:44,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624720 2024-12-10T11:03:44,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624720 2024-12-10T11:03:44,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624720) roll requested 2024-12-10T11:03:44,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624727 2024-12-10T11:03:44,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624727 2024-12-10T11:03:44,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624727) roll requested 2024-12-10T11:03:44,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624734 2024-12-10T11:03:44,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624734 2024-12-10T11:03:44,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624734) roll requested 2024-12-10T11:03:44,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624742 2024-12-10T11:03:44,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624742 2024-12-10T11:03:44,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624742) roll requested 2024-12-10T11:03:44,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624750 2024-12-10T11:03:44,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624750 2024-12-10T11:03:44,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624750) roll requested 2024-12-10T11:03:44,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624757 2024-12-10T11:03:44,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624757 2024-12-10T11:03:44,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624757) roll requested 2024-12-10T11:03:44,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624766 2024-12-10T11:03:44,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624766 2024-12-10T11:03:44,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624766) roll requested 2024-12-10T11:03:44,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624774 2024-12-10T11:03:44,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624774 2024-12-10T11:03:44,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624774) roll requested 2024-12-10T11:03:44,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624782 2024-12-10T11:03:44,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624782 2024-12-10T11:03:44,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624782) roll requested 2024-12-10T11:03:44,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624789 2024-12-10T11:03:44,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624789 2024-12-10T11:03:44,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624789) roll requested 2024-12-10T11:03:44,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624797 2024-12-10T11:03:44,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624797 2024-12-10T11:03:44,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624797) roll requested 2024-12-10T11:03:44,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624806 2024-12-10T11:03:44,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624806 2024-12-10T11:03:44,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624806) roll requested 2024-12-10T11:03:44,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624813 2024-12-10T11:03:44,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624813 2024-12-10T11:03:44,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624813) roll requested 2024-12-10T11:03:44,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624820 2024-12-10T11:03:44,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624820 2024-12-10T11:03:44,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624820) roll requested 2024-12-10T11:03:44,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624828 2024-12-10T11:03:44,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624828 2024-12-10T11:03:44,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624828) roll requested 2024-12-10T11:03:44,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624837 2024-12-10T11:03:44,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624837 2024-12-10T11:03:44,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624837) roll requested 2024-12-10T11:03:44,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624844 2024-12-10T11:03:44,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624844 2024-12-10T11:03:44,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624844) roll requested 2024-12-10T11:03:44,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624852 2024-12-10T11:03:44,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624852 2024-12-10T11:03:44,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624852) roll requested 2024-12-10T11:03:44,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624859 2024-12-10T11:03:44,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624859 2024-12-10T11:03:44,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624859) roll requested 2024-12-10T11:03:44,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624867 2024-12-10T11:03:44,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624867 2024-12-10T11:03:44,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624867) roll requested 2024-12-10T11:03:44,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624874 2024-12-10T11:03:44,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624874 2024-12-10T11:03:44,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624874) roll requested 2024-12-10T11:03:44,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624884 2024-12-10T11:03:44,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624884 2024-12-10T11:03:44,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624884) roll requested 2024-12-10T11:03:44,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624893 2024-12-10T11:03:44,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624893 2024-12-10T11:03:44,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624893) roll requested 2024-12-10T11:03:44,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624901 2024-12-10T11:03:44,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624901 2024-12-10T11:03:44,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624901) roll requested 2024-12-10T11:03:44,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624908 2024-12-10T11:03:44,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624908 2024-12-10T11:03:44,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624908) roll requested 2024-12-10T11:03:44,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624917 2024-12-10T11:03:44,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624917 2024-12-10T11:03:44,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624917) roll requested 2024-12-10T11:03:44,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624924 2024-12-10T11:03:44,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624924 2024-12-10T11:03:44,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624924) roll requested 2024-12-10T11:03:44,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624932 2024-12-10T11:03:44,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624932 2024-12-10T11:03:44,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624932) roll requested 2024-12-10T11:03:44,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624942 2024-12-10T11:03:44,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624942 2024-12-10T11:03:44,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624942) roll requested 2024-12-10T11:03:44,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624951 2024-12-10T11:03:44,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624951 2024-12-10T11:03:44,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624951) roll requested 2024-12-10T11:03:44,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624958 2024-12-10T11:03:44,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624958 2024-12-10T11:03:44,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624958) roll requested 2024-12-10T11:03:44,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624966 2024-12-10T11:03:44,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624966 2024-12-10T11:03:44,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624966) roll requested 2024-12-10T11:03:44,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624974 2024-12-10T11:03:44,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624974 2024-12-10T11:03:44,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624974) roll requested 2024-12-10T11:03:44,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624981 2024-12-10T11:03:44,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624981 2024-12-10T11:03:44,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624981) roll requested 2024-12-10T11:03:44,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624988 2024-12-10T11:03:44,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:44,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624988 2024-12-10T11:03:44,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:44,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:44,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624988) roll requested 2024-12-10T11:03:44,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828624996 2024-12-10T11:03:45,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624996 2024-12-10T11:03:45,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828624996) roll requested 2024-12-10T11:03:45,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625003 2024-12-10T11:03:45,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828624996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625003 2024-12-10T11:03:45,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625003) roll requested 2024-12-10T11:03:45,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625010 2024-12-10T11:03:45,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625010 2024-12-10T11:03:45,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625010) roll requested 2024-12-10T11:03:45,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625017 2024-12-10T11:03:45,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625017 2024-12-10T11:03:45,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625017) roll requested 2024-12-10T11:03:45,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625025 2024-12-10T11:03:45,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625025 2024-12-10T11:03:45,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625025) roll requested 2024-12-10T11:03:45,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625034 2024-12-10T11:03:45,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625034 2024-12-10T11:03:45,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625034) roll requested 2024-12-10T11:03:45,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625041 2024-12-10T11:03:45,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625041 2024-12-10T11:03:45,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625041) roll requested 2024-12-10T11:03:45,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625049 2024-12-10T11:03:45,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625049 2024-12-10T11:03:45,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625049) roll requested 2024-12-10T11:03:45,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625057 2024-12-10T11:03:45,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625057 2024-12-10T11:03:45,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625057) roll requested 2024-12-10T11:03:45,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625064 2024-12-10T11:03:45,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625064 2024-12-10T11:03:45,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625064) roll requested 2024-12-10T11:03:45,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625071 2024-12-10T11:03:45,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625071 2024-12-10T11:03:45,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625071) roll requested 2024-12-10T11:03:45,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625081 2024-12-10T11:03:45,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625081 2024-12-10T11:03:45,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625081) roll requested 2024-12-10T11:03:45,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625089 2024-12-10T11:03:45,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625089 2024-12-10T11:03:45,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625089) roll requested 2024-12-10T11:03:45,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625097 2024-12-10T11:03:45,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625097 2024-12-10T11:03:45,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625097) roll requested 2024-12-10T11:03:45,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625106 2024-12-10T11:03:45,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625106 2024-12-10T11:03:45,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625106) roll requested 2024-12-10T11:03:45,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625113 2024-12-10T11:03:45,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625113 2024-12-10T11:03:45,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625113) roll requested 2024-12-10T11:03:45,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625121 2024-12-10T11:03:45,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625121 2024-12-10T11:03:45,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625121) roll requested 2024-12-10T11:03:45,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625128 2024-12-10T11:03:45,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625128 2024-12-10T11:03:45,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625128) roll requested 2024-12-10T11:03:45,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625137 2024-12-10T11:03:45,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625137 2024-12-10T11:03:45,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625137) roll requested 2024-12-10T11:03:45,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625146 2024-12-10T11:03:45,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625146 2024-12-10T11:03:45,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625146) roll requested 2024-12-10T11:03:45,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625156 2024-12-10T11:03:45,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625156 2024-12-10T11:03:45,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625156) roll requested 2024-12-10T11:03:45,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625165 2024-12-10T11:03:45,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625165 2024-12-10T11:03:45,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625165) roll requested 2024-12-10T11:03:45,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625173 2024-12-10T11:03:45,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625173 2024-12-10T11:03:45,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625173) roll requested 2024-12-10T11:03:45,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625181 2024-12-10T11:03:45,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625181 2024-12-10T11:03:45,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625181) roll requested 2024-12-10T11:03:45,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625190 2024-12-10T11:03:45,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625190 2024-12-10T11:03:45,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625190) roll requested 2024-12-10T11:03:45,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625197 2024-12-10T11:03:45,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625197 2024-12-10T11:03:45,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625197) roll requested 2024-12-10T11:03:45,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625204 2024-12-10T11:03:45,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625204 2024-12-10T11:03:45,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625204) roll requested 2024-12-10T11:03:45,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625212 2024-12-10T11:03:45,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625212 2024-12-10T11:03:45,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625212) roll requested 2024-12-10T11:03:45,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625220 2024-12-10T11:03:45,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625220 2024-12-10T11:03:45,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625220) roll requested 2024-12-10T11:03:45,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625230 2024-12-10T11:03:45,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625230 2024-12-10T11:03:45,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625230) roll requested 2024-12-10T11:03:45,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625237 2024-12-10T11:03:45,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625237 2024-12-10T11:03:45,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625237) roll requested 2024-12-10T11:03:45,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625250 2024-12-10T11:03:45,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625250 2024-12-10T11:03:45,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625250) roll requested 2024-12-10T11:03:45,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625257 2024-12-10T11:03:45,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625257 2024-12-10T11:03:45,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625257) roll requested 2024-12-10T11:03:45,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625265 2024-12-10T11:03:45,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625265 2024-12-10T11:03:45,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625265) roll requested 2024-12-10T11:03:45,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625273 2024-12-10T11:03:45,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625273 2024-12-10T11:03:45,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625273) roll requested 2024-12-10T11:03:45,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625281 2024-12-10T11:03:45,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625281 2024-12-10T11:03:45,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625281) roll requested 2024-12-10T11:03:45,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625288 2024-12-10T11:03:45,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625288 2024-12-10T11:03:45,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625288) roll requested 2024-12-10T11:03:45,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625298 2024-12-10T11:03:45,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625298 2024-12-10T11:03:45,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625298) roll requested 2024-12-10T11:03:45,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625305 2024-12-10T11:03:45,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625305 2024-12-10T11:03:45,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625305) roll requested 2024-12-10T11:03:45,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625313 2024-12-10T11:03:45,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625313 2024-12-10T11:03:45,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625313) roll requested 2024-12-10T11:03:45,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625322 2024-12-10T11:03:45,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625322 2024-12-10T11:03:45,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625322) roll requested 2024-12-10T11:03:45,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625331 2024-12-10T11:03:45,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625331 2024-12-10T11:03:45,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625331) roll requested 2024-12-10T11:03:45,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625339 2024-12-10T11:03:45,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625339 2024-12-10T11:03:45,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625339) roll requested 2024-12-10T11:03:45,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625349 2024-12-10T11:03:45,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625349 2024-12-10T11:03:45,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625349) roll requested 2024-12-10T11:03:45,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625356 2024-12-10T11:03:45,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625356 2024-12-10T11:03:45,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625356) roll requested 2024-12-10T11:03:45,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625363 2024-12-10T11:03:45,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625363 2024-12-10T11:03:45,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625363) roll requested 2024-12-10T11:03:45,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625372 2024-12-10T11:03:45,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625372 2024-12-10T11:03:45,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625372) roll requested 2024-12-10T11:03:45,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625380 2024-12-10T11:03:45,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625380 2024-12-10T11:03:45,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625380) roll requested 2024-12-10T11:03:45,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625388 2024-12-10T11:03:45,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625388 2024-12-10T11:03:45,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625388) roll requested 2024-12-10T11:03:45,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625398 2024-12-10T11:03:45,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625398 2024-12-10T11:03:45,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625398) roll requested 2024-12-10T11:03:45,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625405 2024-12-10T11:03:45,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625405 2024-12-10T11:03:45,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625405) roll requested 2024-12-10T11:03:45,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625414 2024-12-10T11:03:45,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625414 2024-12-10T11:03:45,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625414) roll requested 2024-12-10T11:03:45,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625421 2024-12-10T11:03:45,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625421 2024-12-10T11:03:45,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625421) roll requested 2024-12-10T11:03:45,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625433 2024-12-10T11:03:45,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625433 2024-12-10T11:03:45,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625433) roll requested 2024-12-10T11:03:45,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625441 2024-12-10T11:03:45,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625441 2024-12-10T11:03:45,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625441) roll requested 2024-12-10T11:03:45,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625449 2024-12-10T11:03:45,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625449 2024-12-10T11:03:45,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625449) roll requested 2024-12-10T11:03:45,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625456 2024-12-10T11:03:45,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625456 2024-12-10T11:03:45,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625456) roll requested 2024-12-10T11:03:45,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625463 2024-12-10T11:03:45,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625463 2024-12-10T11:03:45,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625463) roll requested 2024-12-10T11:03:45,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625471 2024-12-10T11:03:45,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625471 2024-12-10T11:03:45,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625471) roll requested 2024-12-10T11:03:45,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625478 2024-12-10T11:03:45,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625478 2024-12-10T11:03:45,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625478) roll requested 2024-12-10T11:03:45,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625487 2024-12-10T11:03:45,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625487 2024-12-10T11:03:45,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625487) roll requested 2024-12-10T11:03:45,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625494 2024-12-10T11:03:45,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625494 2024-12-10T11:03:45,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625494) roll requested 2024-12-10T11:03:45,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625501 2024-12-10T11:03:45,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625501 2024-12-10T11:03:45,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625501) roll requested 2024-12-10T11:03:45,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625513 2024-12-10T11:03:45,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625513 2024-12-10T11:03:45,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625513) roll requested 2024-12-10T11:03:45,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625520 2024-12-10T11:03:45,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625520 2024-12-10T11:03:45,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625520) roll requested 2024-12-10T11:03:45,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625528 2024-12-10T11:03:45,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625528 2024-12-10T11:03:45,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625528) roll requested 2024-12-10T11:03:45,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625536 2024-12-10T11:03:45,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625536 2024-12-10T11:03:45,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625536) roll requested 2024-12-10T11:03:45,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625544 2024-12-10T11:03:45,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625544 2024-12-10T11:03:45,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625544) roll requested 2024-12-10T11:03:45,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625551 2024-12-10T11:03:45,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625551 2024-12-10T11:03:45,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625551) roll requested 2024-12-10T11:03:45,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625562 2024-12-10T11:03:45,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625562 2024-12-10T11:03:45,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625562) roll requested 2024-12-10T11:03:45,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625573 2024-12-10T11:03:45,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625573 2024-12-10T11:03:45,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625573) roll requested 2024-12-10T11:03:45,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625580 2024-12-10T11:03:45,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625580 2024-12-10T11:03:45,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625580) roll requested 2024-12-10T11:03:45,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625587 2024-12-10T11:03:45,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625587 2024-12-10T11:03:45,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625587) roll requested 2024-12-10T11:03:45,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625595 2024-12-10T11:03:45,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625595 2024-12-10T11:03:45,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625595) roll requested 2024-12-10T11:03:45,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625602 2024-12-10T11:03:45,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625602 2024-12-10T11:03:45,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625602) roll requested 2024-12-10T11:03:45,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625608 2024-12-10T11:03:45,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625608 2024-12-10T11:03:45,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625608) roll requested 2024-12-10T11:03:45,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625622 2024-12-10T11:03:45,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625622 2024-12-10T11:03:45,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625622) roll requested 2024-12-10T11:03:45,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625629 2024-12-10T11:03:45,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625629 2024-12-10T11:03:45,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625629) roll requested 2024-12-10T11:03:45,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625636 2024-12-10T11:03:45,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625636 2024-12-10T11:03:45,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625636) roll requested 2024-12-10T11:03:45,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625643 2024-12-10T11:03:45,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625643 2024-12-10T11:03:45,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625643) roll requested 2024-12-10T11:03:45,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625650 2024-12-10T11:03:45,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625650 2024-12-10T11:03:45,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625650) roll requested 2024-12-10T11:03:45,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625658 2024-12-10T11:03:45,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625658 2024-12-10T11:03:45,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625658) roll requested 2024-12-10T11:03:45,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625665 2024-12-10T11:03:45,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625665 2024-12-10T11:03:45,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625665) roll requested 2024-12-10T11:03:45,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625672 2024-12-10T11:03:45,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625672 2024-12-10T11:03:45,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625672) roll requested 2024-12-10T11:03:45,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625679 2024-12-10T11:03:45,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625679 2024-12-10T11:03:45,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625679) roll requested 2024-12-10T11:03:45,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625686 2024-12-10T11:03:45,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625686 2024-12-10T11:03:45,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625686) roll requested 2024-12-10T11:03:45,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625693 2024-12-10T11:03:45,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625693 2024-12-10T11:03:45,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625693) roll requested 2024-12-10T11:03:45,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625700 2024-12-10T11:03:45,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625700 2024-12-10T11:03:45,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625700) roll requested 2024-12-10T11:03:45,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625707 2024-12-10T11:03:45,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625707 2024-12-10T11:03:45,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625707) roll requested 2024-12-10T11:03:45,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625715 2024-12-10T11:03:45,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625715 2024-12-10T11:03:45,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625715) roll requested 2024-12-10T11:03:45,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625722 2024-12-10T11:03:45,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625722 2024-12-10T11:03:45,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625722) roll requested 2024-12-10T11:03:45,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625729 2024-12-10T11:03:45,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625729 2024-12-10T11:03:45,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625729) roll requested 2024-12-10T11:03:45,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625737 2024-12-10T11:03:45,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625737 2024-12-10T11:03:45,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625737) roll requested 2024-12-10T11:03:45,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625745 2024-12-10T11:03:45,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625745 2024-12-10T11:03:45,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625745) roll requested 2024-12-10T11:03:45,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625757 2024-12-10T11:03:45,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625757 2024-12-10T11:03:45,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625757) roll requested 2024-12-10T11:03:45,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625768 2024-12-10T11:03:45,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625768 2024-12-10T11:03:45,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625768) roll requested 2024-12-10T11:03:45,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625779 2024-12-10T11:03:45,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625779 2024-12-10T11:03:45,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625779) roll requested 2024-12-10T11:03:45,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625789 2024-12-10T11:03:45,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625789 2024-12-10T11:03:45,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625789) roll requested 2024-12-10T11:03:45,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625797 2024-12-10T11:03:45,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625797 2024-12-10T11:03:45,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625797) roll requested 2024-12-10T11:03:45,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625807 2024-12-10T11:03:45,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625807 2024-12-10T11:03:45,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625807) roll requested 2024-12-10T11:03:45,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625815 2024-12-10T11:03:45,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625815 2024-12-10T11:03:45,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625815) roll requested 2024-12-10T11:03:45,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625826 2024-12-10T11:03:45,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625826 2024-12-10T11:03:45,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625826) roll requested 2024-12-10T11:03:45,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625834 2024-12-10T11:03:45,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625834 2024-12-10T11:03:45,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625834) roll requested 2024-12-10T11:03:45,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625843 2024-12-10T11:03:45,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625843 2024-12-10T11:03:45,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625843) roll requested 2024-12-10T11:03:45,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625850 2024-12-10T11:03:45,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625850 2024-12-10T11:03:45,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625850) roll requested 2024-12-10T11:03:45,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625856 2024-12-10T11:03:45,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625856 2024-12-10T11:03:45,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625856) roll requested 2024-12-10T11:03:45,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625863 2024-12-10T11:03:45,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625863 2024-12-10T11:03:45,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625863) roll requested 2024-12-10T11:03:45,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625870 2024-12-10T11:03:45,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625870 2024-12-10T11:03:45,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625870) roll requested 2024-12-10T11:03:45,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625878 2024-12-10T11:03:45,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625878 2024-12-10T11:03:45,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625878) roll requested 2024-12-10T11:03:45,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625886 2024-12-10T11:03:45,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625886 2024-12-10T11:03:45,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625886) roll requested 2024-12-10T11:03:45,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625894 2024-12-10T11:03:45,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625894 2024-12-10T11:03:45,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625894) roll requested 2024-12-10T11:03:45,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625901 2024-12-10T11:03:45,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625901 2024-12-10T11:03:45,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625901) roll requested 2024-12-10T11:03:45,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625908 2024-12-10T11:03:45,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625908 2024-12-10T11:03:45,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625908) roll requested 2024-12-10T11:03:45,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625915 2024-12-10T11:03:45,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625915 2024-12-10T11:03:45,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625915) roll requested 2024-12-10T11:03:45,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625922 2024-12-10T11:03:45,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625922 2024-12-10T11:03:45,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625922) roll requested 2024-12-10T11:03:45,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625928 2024-12-10T11:03:45,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625928 2024-12-10T11:03:45,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625928) roll requested 2024-12-10T11:03:45,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625935 2024-12-10T11:03:45,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625935 2024-12-10T11:03:45,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625935) roll requested 2024-12-10T11:03:45,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625942 2024-12-10T11:03:45,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625942 2024-12-10T11:03:45,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625942) roll requested 2024-12-10T11:03:45,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625949 2024-12-10T11:03:45,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625949 2024-12-10T11:03:45,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625949) roll requested 2024-12-10T11:03:45,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625955 2024-12-10T11:03:45,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625955 2024-12-10T11:03:45,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625955) roll requested 2024-12-10T11:03:45,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625962 2024-12-10T11:03:45,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625962 2024-12-10T11:03:45,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625962) roll requested 2024-12-10T11:03:45,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625974 2024-12-10T11:03:45,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625974 2024-12-10T11:03:45,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625974) roll requested 2024-12-10T11:03:45,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625981 2024-12-10T11:03:45,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625981 2024-12-10T11:03:45,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:45,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:45,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625981) roll requested 2024-12-10T11:03:45,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828625988 2024-12-10T11:03:45,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:45,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625988 2024-12-10T11:03:46,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828625988) roll requested 2024-12-10T11:03:46,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626000 2024-12-10T11:03:46,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828625988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626000 2024-12-10T11:03:46,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626000) roll requested 2024-12-10T11:03:46,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626007 2024-12-10T11:03:46,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626007 2024-12-10T11:03:46,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626007) roll requested 2024-12-10T11:03:46,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626016 2024-12-10T11:03:46,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626016 2024-12-10T11:03:46,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626016) roll requested 2024-12-10T11:03:46,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626023 2024-12-10T11:03:46,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626023 2024-12-10T11:03:46,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626023) roll requested 2024-12-10T11:03:46,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626033 2024-12-10T11:03:46,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626033 2024-12-10T11:03:46,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626033) roll requested 2024-12-10T11:03:46,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626040 2024-12-10T11:03:46,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626040 2024-12-10T11:03:46,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626040) roll requested 2024-12-10T11:03:46,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626050 2024-12-10T11:03:46,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626050 2024-12-10T11:03:46,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626050) roll requested 2024-12-10T11:03:46,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626058 2024-12-10T11:03:46,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626058 2024-12-10T11:03:46,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626058) roll requested 2024-12-10T11:03:46,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626066 2024-12-10T11:03:46,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626066 2024-12-10T11:03:46,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626066) roll requested 2024-12-10T11:03:46,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626074 2024-12-10T11:03:46,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626074 2024-12-10T11:03:46,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626074) roll requested 2024-12-10T11:03:46,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626082 2024-12-10T11:03:46,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626082 2024-12-10T11:03:46,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626082) roll requested 2024-12-10T11:03:46,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626093 2024-12-10T11:03:46,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626093 2024-12-10T11:03:46,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626093) roll requested 2024-12-10T11:03:46,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626103 2024-12-10T11:03:46,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626103 2024-12-10T11:03:46,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626103) roll requested 2024-12-10T11:03:46,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626110 2024-12-10T11:03:46,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626110 2024-12-10T11:03:46,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626110) roll requested 2024-12-10T11:03:46,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626117 2024-12-10T11:03:46,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626117 2024-12-10T11:03:46,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626117) roll requested 2024-12-10T11:03:46,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626124 2024-12-10T11:03:46,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626124 2024-12-10T11:03:46,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626124) roll requested 2024-12-10T11:03:46,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626131 2024-12-10T11:03:46,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626131 2024-12-10T11:03:46,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626131) roll requested 2024-12-10T11:03:46,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626140 2024-12-10T11:03:46,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626140 2024-12-10T11:03:46,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626140) roll requested 2024-12-10T11:03:46,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626147 2024-12-10T11:03:46,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626147 2024-12-10T11:03:46,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626147) roll requested 2024-12-10T11:03:46,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626153 2024-12-10T11:03:46,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626153 2024-12-10T11:03:46,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626153) roll requested 2024-12-10T11:03:46,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626161 2024-12-10T11:03:46,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626161 2024-12-10T11:03:46,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626161) roll requested 2024-12-10T11:03:46,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626167 2024-12-10T11:03:46,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626167 2024-12-10T11:03:46,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626167) roll requested 2024-12-10T11:03:46,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626181 2024-12-10T11:03:46,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626181 2024-12-10T11:03:46,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626181) roll requested 2024-12-10T11:03:46,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626193 2024-12-10T11:03:46,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626193 2024-12-10T11:03:46,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626193) roll requested 2024-12-10T11:03:46,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626201 2024-12-10T11:03:46,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626201 2024-12-10T11:03:46,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626201) roll requested 2024-12-10T11:03:46,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626208 2024-12-10T11:03:46,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626208 2024-12-10T11:03:46,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626208) roll requested 2024-12-10T11:03:46,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626216 2024-12-10T11:03:46,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626216 2024-12-10T11:03:46,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626216) roll requested 2024-12-10T11:03:46,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626223 2024-12-10T11:03:46,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626223 2024-12-10T11:03:46,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626223) roll requested 2024-12-10T11:03:46,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626230 2024-12-10T11:03:46,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626230 2024-12-10T11:03:46,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626230) roll requested 2024-12-10T11:03:46,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626236 2024-12-10T11:03:46,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626236 2024-12-10T11:03:46,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626236) roll requested 2024-12-10T11:03:46,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626243 2024-12-10T11:03:46,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626243 2024-12-10T11:03:46,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626243) roll requested 2024-12-10T11:03:46,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626250 2024-12-10T11:03:46,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626250 2024-12-10T11:03:46,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626250) roll requested 2024-12-10T11:03:46,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626263 2024-12-10T11:03:46,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626263 2024-12-10T11:03:46,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626263) roll requested 2024-12-10T11:03:46,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626269 2024-12-10T11:03:46,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626269 2024-12-10T11:03:46,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626269) roll requested 2024-12-10T11:03:46,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626276 2024-12-10T11:03:46,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626276 2024-12-10T11:03:46,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626276) roll requested 2024-12-10T11:03:46,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626283 2024-12-10T11:03:46,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626283 2024-12-10T11:03:46,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626283) roll requested 2024-12-10T11:03:46,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626298 2024-12-10T11:03:46,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626298 2024-12-10T11:03:46,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626298) roll requested 2024-12-10T11:03:46,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626305 2024-12-10T11:03:46,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626305 2024-12-10T11:03:46,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626305) roll requested 2024-12-10T11:03:46,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626312 2024-12-10T11:03:46,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626312 2024-12-10T11:03:46,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626312) roll requested 2024-12-10T11:03:46,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626321 2024-12-10T11:03:46,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626321 2024-12-10T11:03:46,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626321) roll requested 2024-12-10T11:03:46,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626329 2024-12-10T11:03:46,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626329 2024-12-10T11:03:46,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626329) roll requested 2024-12-10T11:03:46,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626336 2024-12-10T11:03:46,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626336 2024-12-10T11:03:46,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626336) roll requested 2024-12-10T11:03:46,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626346 2024-12-10T11:03:46,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626346 2024-12-10T11:03:46,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626346) roll requested 2024-12-10T11:03:46,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626353 2024-12-10T11:03:46,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626353 2024-12-10T11:03:46,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626353) roll requested 2024-12-10T11:03:46,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626362 2024-12-10T11:03:46,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626362 2024-12-10T11:03:46,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626362) roll requested 2024-12-10T11:03:46,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626370 2024-12-10T11:03:46,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626370 2024-12-10T11:03:46,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626370) roll requested 2024-12-10T11:03:46,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626378 2024-12-10T11:03:46,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626378 2024-12-10T11:03:46,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626378) roll requested 2024-12-10T11:03:46,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626385 2024-12-10T11:03:46,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626385 2024-12-10T11:03:46,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626385) roll requested 2024-12-10T11:03:46,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626394 2024-12-10T11:03:46,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626394 2024-12-10T11:03:46,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626394) roll requested 2024-12-10T11:03:46,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626401 2024-12-10T11:03:46,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626401 2024-12-10T11:03:46,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626401) roll requested 2024-12-10T11:03:46,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626408 2024-12-10T11:03:46,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626408 2024-12-10T11:03:46,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626408) roll requested 2024-12-10T11:03:46,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626416 2024-12-10T11:03:46,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626416 2024-12-10T11:03:46,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626416) roll requested 2024-12-10T11:03:46,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626423 2024-12-10T11:03:46,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626423 2024-12-10T11:03:46,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626423) roll requested 2024-12-10T11:03:46,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626437 2024-12-10T11:03:46,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626437 2024-12-10T11:03:46,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626437) roll requested 2024-12-10T11:03:46,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626447 2024-12-10T11:03:46,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626447 2024-12-10T11:03:46,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626447) roll requested 2024-12-10T11:03:46,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626459 2024-12-10T11:03:46,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626459 2024-12-10T11:03:46,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626459) roll requested 2024-12-10T11:03:46,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626466 2024-12-10T11:03:46,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626466 2024-12-10T11:03:46,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626466) roll requested 2024-12-10T11:03:46,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626473 2024-12-10T11:03:46,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626473 2024-12-10T11:03:46,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626473) roll requested 2024-12-10T11:03:46,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626479 2024-12-10T11:03:46,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626479 2024-12-10T11:03:46,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626479) roll requested 2024-12-10T11:03:46,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626486 2024-12-10T11:03:46,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626486 2024-12-10T11:03:46,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626486) roll requested 2024-12-10T11:03:46,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626493 2024-12-10T11:03:46,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626493 2024-12-10T11:03:46,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626493) roll requested 2024-12-10T11:03:46,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626500 2024-12-10T11:03:46,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626500 2024-12-10T11:03:46,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626500) roll requested 2024-12-10T11:03:46,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626507 2024-12-10T11:03:46,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626507 2024-12-10T11:03:46,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626507) roll requested 2024-12-10T11:03:46,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626513 2024-12-10T11:03:46,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626513 2024-12-10T11:03:46,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626513) roll requested 2024-12-10T11:03:46,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626520 2024-12-10T11:03:46,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626520 2024-12-10T11:03:46,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626520) roll requested 2024-12-10T11:03:46,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626527 2024-12-10T11:03:46,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626527 2024-12-10T11:03:46,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626527) roll requested 2024-12-10T11:03:46,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626534 2024-12-10T11:03:46,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626534 2024-12-10T11:03:46,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626534) roll requested 2024-12-10T11:03:46,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626543 2024-12-10T11:03:46,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626543 2024-12-10T11:03:46,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626543) roll requested 2024-12-10T11:03:46,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626551 2024-12-10T11:03:46,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626551 2024-12-10T11:03:46,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626551) roll requested 2024-12-10T11:03:46,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626559 2024-12-10T11:03:46,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626559 2024-12-10T11:03:46,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626559) roll requested 2024-12-10T11:03:46,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626601 2024-12-10T11:03:46,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626601 2024-12-10T11:03:46,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626601) roll requested 2024-12-10T11:03:46,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626609 2024-12-10T11:03:46,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626609 2024-12-10T11:03:46,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626609) roll requested 2024-12-10T11:03:46,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626617 2024-12-10T11:03:46,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626617 2024-12-10T11:03:46,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626617) roll requested 2024-12-10T11:03:46,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626632 2024-12-10T11:03:46,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626632 2024-12-10T11:03:46,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626632) roll requested 2024-12-10T11:03:46,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626650 2024-12-10T11:03:46,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626650 2024-12-10T11:03:46,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626650) roll requested 2024-12-10T11:03:46,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626659 2024-12-10T11:03:46,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626659 2024-12-10T11:03:46,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626659) roll requested 2024-12-10T11:03:46,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626667 2024-12-10T11:03:46,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626667 2024-12-10T11:03:46,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626667) roll requested 2024-12-10T11:03:46,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626677 2024-12-10T11:03:46,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626677 2024-12-10T11:03:46,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626677) roll requested 2024-12-10T11:03:46,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626684 2024-12-10T11:03:46,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626684 2024-12-10T11:03:46,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626684) roll requested 2024-12-10T11:03:46,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626693 2024-12-10T11:03:46,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626693 2024-12-10T11:03:46,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626693) roll requested 2024-12-10T11:03:46,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626700 2024-12-10T11:03:46,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626700 2024-12-10T11:03:46,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626700) roll requested 2024-12-10T11:03:46,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626707 2024-12-10T11:03:46,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626707 2024-12-10T11:03:46,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626707) roll requested 2024-12-10T11:03:46,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626714 2024-12-10T11:03:46,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626714 2024-12-10T11:03:46,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626714) roll requested 2024-12-10T11:03:46,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626722 2024-12-10T11:03:46,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626722 2024-12-10T11:03:46,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626722) roll requested 2024-12-10T11:03:46,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626731 2024-12-10T11:03:46,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626731 2024-12-10T11:03:46,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626731) roll requested 2024-12-10T11:03:46,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626745 2024-12-10T11:03:46,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626745 2024-12-10T11:03:46,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626745) roll requested 2024-12-10T11:03:46,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626758 2024-12-10T11:03:46,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626758 2024-12-10T11:03:46,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626758) roll requested 2024-12-10T11:03:46,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626769 2024-12-10T11:03:46,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626769 2024-12-10T11:03:46,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626769) roll requested 2024-12-10T11:03:46,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626778 2024-12-10T11:03:46,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626778 2024-12-10T11:03:46,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626778) roll requested 2024-12-10T11:03:46,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626785 2024-12-10T11:03:46,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626785 2024-12-10T11:03:46,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626785) roll requested 2024-12-10T11:03:46,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626793 2024-12-10T11:03:46,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626793 2024-12-10T11:03:46,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626793) roll requested 2024-12-10T11:03:46,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626802 2024-12-10T11:03:46,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626802 2024-12-10T11:03:46,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626802) roll requested 2024-12-10T11:03:46,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626814 2024-12-10T11:03:46,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626814 2024-12-10T11:03:46,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626814) roll requested 2024-12-10T11:03:46,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626821 2024-12-10T11:03:46,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626821 2024-12-10T11:03:46,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626821) roll requested 2024-12-10T11:03:46,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626833 2024-12-10T11:03:46,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626833 2024-12-10T11:03:46,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626833) roll requested 2024-12-10T11:03:46,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626839 2024-12-10T11:03:46,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626839 2024-12-10T11:03:46,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626839) roll requested 2024-12-10T11:03:46,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626846 2024-12-10T11:03:46,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626846 2024-12-10T11:03:46,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626846) roll requested 2024-12-10T11:03:46,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626852 2024-12-10T11:03:46,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626852 2024-12-10T11:03:46,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626852) roll requested 2024-12-10T11:03:46,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626859 2024-12-10T11:03:46,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626859 2024-12-10T11:03:46,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626859) roll requested 2024-12-10T11:03:46,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626865 2024-12-10T11:03:46,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626865 2024-12-10T11:03:46,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626865) roll requested 2024-12-10T11:03:46,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626876 2024-12-10T11:03:46,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626876 2024-12-10T11:03:46,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626876) roll requested 2024-12-10T11:03:46,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626885 2024-12-10T11:03:46,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626885 2024-12-10T11:03:46,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626885) roll requested 2024-12-10T11:03:46,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626895 2024-12-10T11:03:46,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626895 2024-12-10T11:03:46,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626895) roll requested 2024-12-10T11:03:46,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626903 2024-12-10T11:03:46,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626903 2024-12-10T11:03:46,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626903) roll requested 2024-12-10T11:03:46,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626912 2024-12-10T11:03:46,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626912 2024-12-10T11:03:46,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626912) roll requested 2024-12-10T11:03:46,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626919 2024-12-10T11:03:46,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626919 2024-12-10T11:03:46,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626919) roll requested 2024-12-10T11:03:46,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626926 2024-12-10T11:03:46,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626926 2024-12-10T11:03:46,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626926) roll requested 2024-12-10T11:03:46,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626932 2024-12-10T11:03:46,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626932 2024-12-10T11:03:46,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626932) roll requested 2024-12-10T11:03:46,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626939 2024-12-10T11:03:46,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626939 2024-12-10T11:03:46,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626939) roll requested 2024-12-10T11:03:46,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626946 2024-12-10T11:03:46,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626946 2024-12-10T11:03:46,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626946) roll requested 2024-12-10T11:03:46,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626952 2024-12-10T11:03:46,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626952 2024-12-10T11:03:46,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626952) roll requested 2024-12-10T11:03:46,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626959 2024-12-10T11:03:46,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626959 2024-12-10T11:03:46,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626959) roll requested 2024-12-10T11:03:46,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626966 2024-12-10T11:03:46,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626966 2024-12-10T11:03:46,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626966) roll requested 2024-12-10T11:03:46,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626972 2024-12-10T11:03:46,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626972 2024-12-10T11:03:46,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626972) roll requested 2024-12-10T11:03:46,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626979 2024-12-10T11:03:46,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626979 2024-12-10T11:03:46,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626979) roll requested 2024-12-10T11:03:46,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626986 2024-12-10T11:03:46,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626986 2024-12-10T11:03:46,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626986) roll requested 2024-12-10T11:03:46,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626992 2024-12-10T11:03:46,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:46,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626992 2024-12-10T11:03:46,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:46,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:46,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626992) roll requested 2024-12-10T11:03:46,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828626999 2024-12-10T11:03:47,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626999 2024-12-10T11:03:47,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828626999) roll requested 2024-12-10T11:03:47,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627005 2024-12-10T11:03:47,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828626999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627005 2024-12-10T11:03:47,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627005) roll requested 2024-12-10T11:03:47,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627012 2024-12-10T11:03:47,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627012 2024-12-10T11:03:47,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627012) roll requested 2024-12-10T11:03:47,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627018 2024-12-10T11:03:47,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627018 2024-12-10T11:03:47,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627018) roll requested 2024-12-10T11:03:47,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627025 2024-12-10T11:03:47,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627025 2024-12-10T11:03:47,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627025) roll requested 2024-12-10T11:03:47,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627031 2024-12-10T11:03:47,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627031 2024-12-10T11:03:47,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627031) roll requested 2024-12-10T11:03:47,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627038 2024-12-10T11:03:47,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627038 2024-12-10T11:03:47,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627038) roll requested 2024-12-10T11:03:47,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627045 2024-12-10T11:03:47,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627045 2024-12-10T11:03:47,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627045) roll requested 2024-12-10T11:03:47,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627051 2024-12-10T11:03:47,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627051 2024-12-10T11:03:47,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627051) roll requested 2024-12-10T11:03:47,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627058 2024-12-10T11:03:47,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627058 2024-12-10T11:03:47,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627058) roll requested 2024-12-10T11:03:47,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627064 2024-12-10T11:03:47,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627064 2024-12-10T11:03:47,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627064) roll requested 2024-12-10T11:03:47,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627071 2024-12-10T11:03:47,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627071 2024-12-10T11:03:47,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627071) roll requested 2024-12-10T11:03:47,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627077 2024-12-10T11:03:47,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627077 2024-12-10T11:03:47,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627077) roll requested 2024-12-10T11:03:47,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627084 2024-12-10T11:03:47,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627084 2024-12-10T11:03:47,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627084) roll requested 2024-12-10T11:03:47,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627093 2024-12-10T11:03:47,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627093 2024-12-10T11:03:47,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627093) roll requested 2024-12-10T11:03:47,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627100 2024-12-10T11:03:47,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627100 2024-12-10T11:03:47,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627100) roll requested 2024-12-10T11:03:47,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627107 2024-12-10T11:03:47,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627107 2024-12-10T11:03:47,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627107) roll requested 2024-12-10T11:03:47,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627113 2024-12-10T11:03:47,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627113 2024-12-10T11:03:47,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627113) roll requested 2024-12-10T11:03:47,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627120 2024-12-10T11:03:47,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627120 2024-12-10T11:03:47,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627120) roll requested 2024-12-10T11:03:47,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627127 2024-12-10T11:03:47,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627127 2024-12-10T11:03:47,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627127) roll requested 2024-12-10T11:03:47,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627133 2024-12-10T11:03:47,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627133 2024-12-10T11:03:47,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627133) roll requested 2024-12-10T11:03:47,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627139 2024-12-10T11:03:47,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627139 2024-12-10T11:03:47,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627139) roll requested 2024-12-10T11:03:47,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627146 2024-12-10T11:03:47,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627146 2024-12-10T11:03:47,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627146) roll requested 2024-12-10T11:03:47,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627153 2024-12-10T11:03:47,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627153 2024-12-10T11:03:47,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627153) roll requested 2024-12-10T11:03:47,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627161 2024-12-10T11:03:47,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627161 2024-12-10T11:03:47,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627161) roll requested 2024-12-10T11:03:47,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627168 2024-12-10T11:03:47,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627168 2024-12-10T11:03:47,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627168) roll requested 2024-12-10T11:03:47,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627180 2024-12-10T11:03:47,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627180 2024-12-10T11:03:47,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627180) roll requested 2024-12-10T11:03:47,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627189 2024-12-10T11:03:47,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627189 2024-12-10T11:03:47,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627189) roll requested 2024-12-10T11:03:47,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627196 2024-12-10T11:03:47,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627196 2024-12-10T11:03:47,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627196) roll requested 2024-12-10T11:03:47,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627204 2024-12-10T11:03:47,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627204 2024-12-10T11:03:47,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627204) roll requested 2024-12-10T11:03:47,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627210 2024-12-10T11:03:47,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627210 2024-12-10T11:03:47,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627210) roll requested 2024-12-10T11:03:47,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627217 2024-12-10T11:03:47,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627217 2024-12-10T11:03:47,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627217) roll requested 2024-12-10T11:03:47,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627225 2024-12-10T11:03:47,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627225 2024-12-10T11:03:47,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627225) roll requested 2024-12-10T11:03:47,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627235 2024-12-10T11:03:47,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627235 2024-12-10T11:03:47,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627235) roll requested 2024-12-10T11:03:47,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627242 2024-12-10T11:03:47,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627242 2024-12-10T11:03:47,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627242) roll requested 2024-12-10T11:03:47,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627248 2024-12-10T11:03:47,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627248 2024-12-10T11:03:47,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627248) roll requested 2024-12-10T11:03:47,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627255 2024-12-10T11:03:47,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627255 2024-12-10T11:03:47,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627255) roll requested 2024-12-10T11:03:47,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627262 2024-12-10T11:03:47,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627262 2024-12-10T11:03:47,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627262) roll requested 2024-12-10T11:03:47,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627268 2024-12-10T11:03:47,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627268 2024-12-10T11:03:47,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627268) roll requested 2024-12-10T11:03:47,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627274 2024-12-10T11:03:47,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627274 2024-12-10T11:03:47,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627274) roll requested 2024-12-10T11:03:47,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627281 2024-12-10T11:03:47,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627281 2024-12-10T11:03:47,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627281) roll requested 2024-12-10T11:03:47,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627288 2024-12-10T11:03:47,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627288 2024-12-10T11:03:47,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627288) roll requested 2024-12-10T11:03:47,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627294 2024-12-10T11:03:47,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627294 2024-12-10T11:03:47,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627294) roll requested 2024-12-10T11:03:47,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627301 2024-12-10T11:03:47,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627301 2024-12-10T11:03:47,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627301) roll requested 2024-12-10T11:03:47,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627307 2024-12-10T11:03:47,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627307 2024-12-10T11:03:47,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627307) roll requested 2024-12-10T11:03:47,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627314 2024-12-10T11:03:47,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627314 2024-12-10T11:03:47,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627314) roll requested 2024-12-10T11:03:47,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627321 2024-12-10T11:03:47,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627321 2024-12-10T11:03:47,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627321) roll requested 2024-12-10T11:03:47,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627327 2024-12-10T11:03:47,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627327 2024-12-10T11:03:47,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627327) roll requested 2024-12-10T11:03:47,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627334 2024-12-10T11:03:47,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627334 2024-12-10T11:03:47,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627334) roll requested 2024-12-10T11:03:47,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627341 2024-12-10T11:03:47,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627341 2024-12-10T11:03:47,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627341) roll requested 2024-12-10T11:03:47,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627348 2024-12-10T11:03:47,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627348 2024-12-10T11:03:47,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627348) roll requested 2024-12-10T11:03:47,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627354 2024-12-10T11:03:47,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627354 2024-12-10T11:03:47,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627354) roll requested 2024-12-10T11:03:47,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627361 2024-12-10T11:03:47,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627361 2024-12-10T11:03:47,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627361) roll requested 2024-12-10T11:03:47,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627367 2024-12-10T11:03:47,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627367 2024-12-10T11:03:47,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627367) roll requested 2024-12-10T11:03:47,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627374 2024-12-10T11:03:47,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627374 2024-12-10T11:03:47,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627374) roll requested 2024-12-10T11:03:47,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627380 2024-12-10T11:03:47,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627380 2024-12-10T11:03:47,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627380) roll requested 2024-12-10T11:03:47,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627387 2024-12-10T11:03:47,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627387 2024-12-10T11:03:47,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627387) roll requested 2024-12-10T11:03:47,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627393 2024-12-10T11:03:47,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627393 2024-12-10T11:03:47,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627393) roll requested 2024-12-10T11:03:47,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627400 2024-12-10T11:03:47,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627400 2024-12-10T11:03:47,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627400) roll requested 2024-12-10T11:03:47,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627407 2024-12-10T11:03:47,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627407 2024-12-10T11:03:47,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627407) roll requested 2024-12-10T11:03:47,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627414 2024-12-10T11:03:47,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627414 2024-12-10T11:03:47,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627414) roll requested 2024-12-10T11:03:47,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627421 2024-12-10T11:03:47,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627421 2024-12-10T11:03:47,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627421) roll requested 2024-12-10T11:03:47,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627428 2024-12-10T11:03:47,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627428 2024-12-10T11:03:47,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627428) roll requested 2024-12-10T11:03:47,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627436 2024-12-10T11:03:47,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627436 2024-12-10T11:03:47,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627436) roll requested 2024-12-10T11:03:47,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627446 2024-12-10T11:03:47,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627446 2024-12-10T11:03:47,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627446) roll requested 2024-12-10T11:03:47,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627457 2024-12-10T11:03:47,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627457 2024-12-10T11:03:47,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627457) roll requested 2024-12-10T11:03:47,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627466 2024-12-10T11:03:47,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627466 2024-12-10T11:03:47,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627466) roll requested 2024-12-10T11:03:47,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627473 2024-12-10T11:03:47,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627473 2024-12-10T11:03:47,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627473) roll requested 2024-12-10T11:03:47,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627484 2024-12-10T11:03:47,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627484 2024-12-10T11:03:47,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627484) roll requested 2024-12-10T11:03:47,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627493 2024-12-10T11:03:47,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627493 2024-12-10T11:03:47,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627493) roll requested 2024-12-10T11:03:47,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627500 2024-12-10T11:03:47,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627500 2024-12-10T11:03:47,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627500) roll requested 2024-12-10T11:03:47,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627509 2024-12-10T11:03:47,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627509 2024-12-10T11:03:47,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627509) roll requested 2024-12-10T11:03:47,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627517 2024-12-10T11:03:47,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627517 2024-12-10T11:03:47,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627517) roll requested 2024-12-10T11:03:47,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627527 2024-12-10T11:03:47,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627527 2024-12-10T11:03:47,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627527) roll requested 2024-12-10T11:03:47,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627537 2024-12-10T11:03:47,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627537 2024-12-10T11:03:47,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627537) roll requested 2024-12-10T11:03:47,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627543 2024-12-10T11:03:47,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627543 2024-12-10T11:03:47,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627543) roll requested 2024-12-10T11:03:47,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627553 2024-12-10T11:03:47,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627553 2024-12-10T11:03:47,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627553) roll requested 2024-12-10T11:03:47,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627560 2024-12-10T11:03:47,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627560 2024-12-10T11:03:47,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627560) roll requested 2024-12-10T11:03:47,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627567 2024-12-10T11:03:47,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627567 2024-12-10T11:03:47,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627567) roll requested 2024-12-10T11:03:47,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627573 2024-12-10T11:03:47,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627573 2024-12-10T11:03:47,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627573) roll requested 2024-12-10T11:03:47,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627580 2024-12-10T11:03:47,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627580 2024-12-10T11:03:47,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627580) roll requested 2024-12-10T11:03:47,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627592 2024-12-10T11:03:47,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627592 2024-12-10T11:03:47,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627592) roll requested 2024-12-10T11:03:47,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627598 2024-12-10T11:03:47,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627598 2024-12-10T11:03:47,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627598) roll requested 2024-12-10T11:03:47,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627609 2024-12-10T11:03:47,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627609 2024-12-10T11:03:47,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627609) roll requested 2024-12-10T11:03:47,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627622 2024-12-10T11:03:47,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627622 2024-12-10T11:03:47,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627622) roll requested 2024-12-10T11:03:47,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627630 2024-12-10T11:03:47,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627630 2024-12-10T11:03:47,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627630) roll requested 2024-12-10T11:03:47,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627641 2024-12-10T11:03:47,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627641 2024-12-10T11:03:47,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627641) roll requested 2024-12-10T11:03:47,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627651 2024-12-10T11:03:47,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627651 2024-12-10T11:03:47,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627651) roll requested 2024-12-10T11:03:47,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627659 2024-12-10T11:03:47,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627659 2024-12-10T11:03:47,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627659) roll requested 2024-12-10T11:03:47,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627666 2024-12-10T11:03:47,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627666 2024-12-10T11:03:47,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627666) roll requested 2024-12-10T11:03:47,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627673 2024-12-10T11:03:47,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627673 2024-12-10T11:03:47,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627673) roll requested 2024-12-10T11:03:47,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627686 2024-12-10T11:03:47,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627686 2024-12-10T11:03:47,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627686) roll requested 2024-12-10T11:03:47,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627697 2024-12-10T11:03:47,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627697 2024-12-10T11:03:47,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627697) roll requested 2024-12-10T11:03:47,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627705 2024-12-10T11:03:47,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627705 2024-12-10T11:03:47,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627705) roll requested 2024-12-10T11:03:47,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627711 2024-12-10T11:03:47,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627711 2024-12-10T11:03:47,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627711) roll requested 2024-12-10T11:03:47,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627719 2024-12-10T11:03:47,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627719 2024-12-10T11:03:47,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627719) roll requested 2024-12-10T11:03:47,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627732 2024-12-10T11:03:47,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627732 2024-12-10T11:03:47,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627732) roll requested 2024-12-10T11:03:47,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627739 2024-12-10T11:03:47,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627739 2024-12-10T11:03:47,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627739) roll requested 2024-12-10T11:03:47,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627746 2024-12-10T11:03:47,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627746 2024-12-10T11:03:47,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627746) roll requested 2024-12-10T11:03:47,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627754 2024-12-10T11:03:47,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627754 2024-12-10T11:03:47,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627754) roll requested 2024-12-10T11:03:47,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627761 2024-12-10T11:03:47,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627761 2024-12-10T11:03:47,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627761) roll requested 2024-12-10T11:03:47,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627769 2024-12-10T11:03:47,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627769 2024-12-10T11:03:47,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627769) roll requested 2024-12-10T11:03:47,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627776 2024-12-10T11:03:47,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627776 2024-12-10T11:03:47,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627776) roll requested 2024-12-10T11:03:47,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627783 2024-12-10T11:03:47,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627783 2024-12-10T11:03:47,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627783) roll requested 2024-12-10T11:03:47,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627790 2024-12-10T11:03:47,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627790 2024-12-10T11:03:47,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627790) roll requested 2024-12-10T11:03:47,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627799 2024-12-10T11:03:47,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627799 2024-12-10T11:03:47,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627799) roll requested 2024-12-10T11:03:47,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627807 2024-12-10T11:03:47,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627807 2024-12-10T11:03:47,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627807) roll requested 2024-12-10T11:03:47,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627814 2024-12-10T11:03:47,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627814 2024-12-10T11:03:47,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627814) roll requested 2024-12-10T11:03:47,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627826 2024-12-10T11:03:47,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627826 2024-12-10T11:03:47,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627826) roll requested 2024-12-10T11:03:47,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627832 2024-12-10T11:03:47,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627832 2024-12-10T11:03:47,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627832) roll requested 2024-12-10T11:03:47,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627839 2024-12-10T11:03:47,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627839 2024-12-10T11:03:47,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627839) roll requested 2024-12-10T11:03:47,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627845 2024-12-10T11:03:47,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627845 2024-12-10T11:03:47,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627845) roll requested 2024-12-10T11:03:47,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627852 2024-12-10T11:03:47,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627852 2024-12-10T11:03:47,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627852) roll requested 2024-12-10T11:03:47,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627861 2024-12-10T11:03:47,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627861 2024-12-10T11:03:47,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627861) roll requested 2024-12-10T11:03:47,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627871 2024-12-10T11:03:47,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627871 2024-12-10T11:03:47,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627871) roll requested 2024-12-10T11:03:47,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627878 2024-12-10T11:03:47,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627878 2024-12-10T11:03:47,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627878) roll requested 2024-12-10T11:03:47,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627887 2024-12-10T11:03:47,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627887 2024-12-10T11:03:47,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627887) roll requested 2024-12-10T11:03:47,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627894 2024-12-10T11:03:47,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627894 2024-12-10T11:03:47,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627894) roll requested 2024-12-10T11:03:47,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627902 2024-12-10T11:03:47,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627902 2024-12-10T11:03:47,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627902) roll requested 2024-12-10T11:03:47,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627914 2024-12-10T11:03:47,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627914 2024-12-10T11:03:47,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627914) roll requested 2024-12-10T11:03:47,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627921 2024-12-10T11:03:47,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627921 2024-12-10T11:03:47,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627921) roll requested 2024-12-10T11:03:47,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627928 2024-12-10T11:03:47,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627928 2024-12-10T11:03:47,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627928) roll requested 2024-12-10T11:03:47,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627934 2024-12-10T11:03:47,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627934 2024-12-10T11:03:47,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627934) roll requested 2024-12-10T11:03:47,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627940 2024-12-10T11:03:47,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627940 2024-12-10T11:03:47,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627940) roll requested 2024-12-10T11:03:47,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627947 2024-12-10T11:03:47,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627947 2024-12-10T11:03:47,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627947) roll requested 2024-12-10T11:03:47,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627953 2024-12-10T11:03:47,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627953 2024-12-10T11:03:47,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627953) roll requested 2024-12-10T11:03:47,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627959 2024-12-10T11:03:47,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627959 2024-12-10T11:03:47,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627959) roll requested 2024-12-10T11:03:47,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627967 2024-12-10T11:03:47,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627967 2024-12-10T11:03:47,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627967) roll requested 2024-12-10T11:03:47,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627973 2024-12-10T11:03:47,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627973 2024-12-10T11:03:47,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627973) roll requested 2024-12-10T11:03:47,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627979 2024-12-10T11:03:47,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627979 2024-12-10T11:03:47,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627979) roll requested 2024-12-10T11:03:47,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627986 2024-12-10T11:03:47,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627986 2024-12-10T11:03:47,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627986) roll requested 2024-12-10T11:03:47,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627992 2024-12-10T11:03:47,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:47,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627992 2024-12-10T11:03:47,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:47,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:47,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627992) roll requested 2024-12-10T11:03:47,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828627999 2024-12-10T11:03:48,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627999 2024-12-10T11:03:48,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828627999) roll requested 2024-12-10T11:03:48,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628005 2024-12-10T11:03:48,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828627999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628005 2024-12-10T11:03:48,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628005) roll requested 2024-12-10T11:03:48,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628012 2024-12-10T11:03:48,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628012 2024-12-10T11:03:48,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628012) roll requested 2024-12-10T11:03:48,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628019 2024-12-10T11:03:48,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628019 2024-12-10T11:03:48,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628019) roll requested 2024-12-10T11:03:48,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628025 2024-12-10T11:03:48,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628025 2024-12-10T11:03:48,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628025) roll requested 2024-12-10T11:03:48,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628031 2024-12-10T11:03:48,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628031 2024-12-10T11:03:48,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628031) roll requested 2024-12-10T11:03:48,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628038 2024-12-10T11:03:48,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628038 2024-12-10T11:03:48,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628038) roll requested 2024-12-10T11:03:48,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628044 2024-12-10T11:03:48,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628044 2024-12-10T11:03:48,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628044) roll requested 2024-12-10T11:03:48,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628051 2024-12-10T11:03:48,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628051 2024-12-10T11:03:48,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628051) roll requested 2024-12-10T11:03:48,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628058 2024-12-10T11:03:48,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628058 2024-12-10T11:03:48,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628058) roll requested 2024-12-10T11:03:48,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628065 2024-12-10T11:03:48,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628065 2024-12-10T11:03:48,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628065) roll requested 2024-12-10T11:03:48,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628071 2024-12-10T11:03:48,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628071 2024-12-10T11:03:48,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628071) roll requested 2024-12-10T11:03:48,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628077 2024-12-10T11:03:48,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628077 2024-12-10T11:03:48,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628077) roll requested 2024-12-10T11:03:48,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628084 2024-12-10T11:03:48,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628084 2024-12-10T11:03:48,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628084) roll requested 2024-12-10T11:03:48,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628090 2024-12-10T11:03:48,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628090 2024-12-10T11:03:48,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628090) roll requested 2024-12-10T11:03:48,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628097 2024-12-10T11:03:48,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628097 2024-12-10T11:03:48,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628097) roll requested 2024-12-10T11:03:48,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628103 2024-12-10T11:03:48,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628103 2024-12-10T11:03:48,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628103) roll requested 2024-12-10T11:03:48,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628110 2024-12-10T11:03:48,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628110 2024-12-10T11:03:48,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628110) roll requested 2024-12-10T11:03:48,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628116 2024-12-10T11:03:48,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628116 2024-12-10T11:03:48,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628116) roll requested 2024-12-10T11:03:48,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628126 2024-12-10T11:03:48,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628126 2024-12-10T11:03:48,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628126) roll requested 2024-12-10T11:03:48,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628133 2024-12-10T11:03:48,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628133 2024-12-10T11:03:48,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628133) roll requested 2024-12-10T11:03:48,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628145 2024-12-10T11:03:48,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628145 2024-12-10T11:03:48,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628145) roll requested 2024-12-10T11:03:48,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628153 2024-12-10T11:03:48,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628153 2024-12-10T11:03:48,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628153) roll requested 2024-12-10T11:03:48,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628160 2024-12-10T11:03:48,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628160 2024-12-10T11:03:48,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628160) roll requested 2024-12-10T11:03:48,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628171 2024-12-10T11:03:48,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628171 2024-12-10T11:03:48,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628171) roll requested 2024-12-10T11:03:48,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628178 2024-12-10T11:03:48,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628178 2024-12-10T11:03:48,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628178) roll requested 2024-12-10T11:03:48,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628184 2024-12-10T11:03:48,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628184 2024-12-10T11:03:48,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628184) roll requested 2024-12-10T11:03:48,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628190 2024-12-10T11:03:48,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628190 2024-12-10T11:03:48,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628190) roll requested 2024-12-10T11:03:48,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628197 2024-12-10T11:03:48,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628197 2024-12-10T11:03:48,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628197) roll requested 2024-12-10T11:03:48,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628204 2024-12-10T11:03:48,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628204 2024-12-10T11:03:48,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628204) roll requested 2024-12-10T11:03:48,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628215 2024-12-10T11:03:48,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628215 2024-12-10T11:03:48,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628215) roll requested 2024-12-10T11:03:48,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628221 2024-12-10T11:03:48,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628221 2024-12-10T11:03:48,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628221) roll requested 2024-12-10T11:03:48,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628227 2024-12-10T11:03:48,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628227 2024-12-10T11:03:48,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628227) roll requested 2024-12-10T11:03:48,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628234 2024-12-10T11:03:48,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628234 2024-12-10T11:03:48,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628234) roll requested 2024-12-10T11:03:48,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628243 2024-12-10T11:03:48,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628243 2024-12-10T11:03:48,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628243) roll requested 2024-12-10T11:03:48,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628250 2024-12-10T11:03:48,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628250 2024-12-10T11:03:48,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628250) roll requested 2024-12-10T11:03:48,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628256 2024-12-10T11:03:48,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628256 2024-12-10T11:03:48,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628256) roll requested 2024-12-10T11:03:48,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628263 2024-12-10T11:03:48,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628263 2024-12-10T11:03:48,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628263) roll requested 2024-12-10T11:03:48,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628269 2024-12-10T11:03:48,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628269 2024-12-10T11:03:48,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628269) roll requested 2024-12-10T11:03:48,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628275 2024-12-10T11:03:48,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628275 2024-12-10T11:03:48,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628275) roll requested 2024-12-10T11:03:48,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628281 2024-12-10T11:03:48,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628281 2024-12-10T11:03:48,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628281) roll requested 2024-12-10T11:03:48,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628287 2024-12-10T11:03:48,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628287 2024-12-10T11:03:48,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628287) roll requested 2024-12-10T11:03:48,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628294 2024-12-10T11:03:48,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628294 2024-12-10T11:03:48,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628294) roll requested 2024-12-10T11:03:48,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628300 2024-12-10T11:03:48,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628300 2024-12-10T11:03:48,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628300) roll requested 2024-12-10T11:03:48,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628306 2024-12-10T11:03:48,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628306 2024-12-10T11:03:48,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628306) roll requested 2024-12-10T11:03:48,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628314 2024-12-10T11:03:48,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628314 2024-12-10T11:03:48,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628314) roll requested 2024-12-10T11:03:48,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628320 2024-12-10T11:03:48,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628320 2024-12-10T11:03:48,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628320) roll requested 2024-12-10T11:03:48,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628331 2024-12-10T11:03:48,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628331 2024-12-10T11:03:48,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628331) roll requested 2024-12-10T11:03:48,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628339 2024-12-10T11:03:48,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628339 2024-12-10T11:03:48,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628339) roll requested 2024-12-10T11:03:48,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628345 2024-12-10T11:03:48,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628345 2024-12-10T11:03:48,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628345) roll requested 2024-12-10T11:03:48,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628352 2024-12-10T11:03:48,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628352 2024-12-10T11:03:48,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628352) roll requested 2024-12-10T11:03:48,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628359 2024-12-10T11:03:48,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628359 2024-12-10T11:03:48,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628359) roll requested 2024-12-10T11:03:48,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628366 2024-12-10T11:03:48,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628366 2024-12-10T11:03:48,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628366) roll requested 2024-12-10T11:03:48,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628372 2024-12-10T11:03:48,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628372 2024-12-10T11:03:48,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628372) roll requested 2024-12-10T11:03:48,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628378 2024-12-10T11:03:48,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628378 2024-12-10T11:03:48,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628378) roll requested 2024-12-10T11:03:48,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628385 2024-12-10T11:03:48,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628385 2024-12-10T11:03:48,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628385) roll requested 2024-12-10T11:03:48,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628391 2024-12-10T11:03:48,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628391 2024-12-10T11:03:48,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628391) roll requested 2024-12-10T11:03:48,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628397 2024-12-10T11:03:48,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628397 2024-12-10T11:03:48,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628397) roll requested 2024-12-10T11:03:48,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628403 2024-12-10T11:03:48,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628403 2024-12-10T11:03:48,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628403) roll requested 2024-12-10T11:03:48,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628410 2024-12-10T11:03:48,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628410 2024-12-10T11:03:48,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628410) roll requested 2024-12-10T11:03:48,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628416 2024-12-10T11:03:48,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628416 2024-12-10T11:03:48,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628416) roll requested 2024-12-10T11:03:48,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628422 2024-12-10T11:03:48,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628422 2024-12-10T11:03:48,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628422) roll requested 2024-12-10T11:03:48,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628429 2024-12-10T11:03:48,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628429 2024-12-10T11:03:48,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628429) roll requested 2024-12-10T11:03:48,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628435 2024-12-10T11:03:48,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628435 2024-12-10T11:03:48,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628435) roll requested 2024-12-10T11:03:48,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628441 2024-12-10T11:03:48,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628441 2024-12-10T11:03:48,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628441) roll requested 2024-12-10T11:03:48,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628448 2024-12-10T11:03:48,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628448 2024-12-10T11:03:48,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628448) roll requested 2024-12-10T11:03:48,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628454 2024-12-10T11:03:48,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628454 2024-12-10T11:03:48,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628454) roll requested 2024-12-10T11:03:48,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628461 2024-12-10T11:03:48,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628461 2024-12-10T11:03:48,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628461) roll requested 2024-12-10T11:03:48,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628467 2024-12-10T11:03:48,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628467 2024-12-10T11:03:48,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628467) roll requested 2024-12-10T11:03:48,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628473 2024-12-10T11:03:48,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628473 2024-12-10T11:03:48,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628473) roll requested 2024-12-10T11:03:48,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628480 2024-12-10T11:03:48,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628480 2024-12-10T11:03:48,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628480) roll requested 2024-12-10T11:03:48,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628486 2024-12-10T11:03:48,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628486 2024-12-10T11:03:48,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628486) roll requested 2024-12-10T11:03:48,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628492 2024-12-10T11:03:48,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628492 2024-12-10T11:03:48,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628492) roll requested 2024-12-10T11:03:48,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628499 2024-12-10T11:03:48,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628499 2024-12-10T11:03:48,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628499) roll requested 2024-12-10T11:03:48,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628505 2024-12-10T11:03:48,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628505 2024-12-10T11:03:48,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628505) roll requested 2024-12-10T11:03:48,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628511 2024-12-10T11:03:48,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628511 2024-12-10T11:03:48,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628511) roll requested 2024-12-10T11:03:48,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628517 2024-12-10T11:03:48,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628517 2024-12-10T11:03:48,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628517) roll requested 2024-12-10T11:03:48,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628524 2024-12-10T11:03:48,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628524 2024-12-10T11:03:48,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628524) roll requested 2024-12-10T11:03:48,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628530 2024-12-10T11:03:48,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628530 2024-12-10T11:03:48,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628530) roll requested 2024-12-10T11:03:48,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628536 2024-12-10T11:03:48,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628536 2024-12-10T11:03:48,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628536) roll requested 2024-12-10T11:03:48,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628543 2024-12-10T11:03:48,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628543 2024-12-10T11:03:48,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628543) roll requested 2024-12-10T11:03:48,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628550 2024-12-10T11:03:48,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628550 2024-12-10T11:03:48,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628550) roll requested 2024-12-10T11:03:48,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628557 2024-12-10T11:03:48,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628557 2024-12-10T11:03:48,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628557) roll requested 2024-12-10T11:03:48,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628565 2024-12-10T11:03:48,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628565 2024-12-10T11:03:48,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628565) roll requested 2024-12-10T11:03:48,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628572 2024-12-10T11:03:48,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628572 2024-12-10T11:03:48,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628572) roll requested 2024-12-10T11:03:48,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628578 2024-12-10T11:03:48,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628578 2024-12-10T11:03:48,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628578) roll requested 2024-12-10T11:03:48,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628585 2024-12-10T11:03:48,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628585 2024-12-10T11:03:48,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628585) roll requested 2024-12-10T11:03:48,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628592 2024-12-10T11:03:48,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628592 2024-12-10T11:03:48,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628592) roll requested 2024-12-10T11:03:48,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628599 2024-12-10T11:03:48,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628599 2024-12-10T11:03:48,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628599) roll requested 2024-12-10T11:03:48,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628605 2024-12-10T11:03:48,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628605 2024-12-10T11:03:48,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628605) roll requested 2024-12-10T11:03:48,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628612 2024-12-10T11:03:48,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628612 2024-12-10T11:03:48,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628612) roll requested 2024-12-10T11:03:48,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628619 2024-12-10T11:03:48,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628619 2024-12-10T11:03:48,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628619) roll requested 2024-12-10T11:03:48,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628625 2024-12-10T11:03:48,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628625 2024-12-10T11:03:48,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628625) roll requested 2024-12-10T11:03:48,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628631 2024-12-10T11:03:48,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628631 2024-12-10T11:03:48,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628631) roll requested 2024-12-10T11:03:48,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628639 2024-12-10T11:03:48,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628639 2024-12-10T11:03:48,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628639) roll requested 2024-12-10T11:03:48,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628645 2024-12-10T11:03:48,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628645 2024-12-10T11:03:48,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628645) roll requested 2024-12-10T11:03:48,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628651 2024-12-10T11:03:48,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628651 2024-12-10T11:03:48,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628651) roll requested 2024-12-10T11:03:48,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628661 2024-12-10T11:03:48,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628661 2024-12-10T11:03:48,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628661) roll requested 2024-12-10T11:03:48,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628668 2024-12-10T11:03:48,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628668 2024-12-10T11:03:48,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628668) roll requested 2024-12-10T11:03:48,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628677 2024-12-10T11:03:48,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628677 2024-12-10T11:03:48,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628677) roll requested 2024-12-10T11:03:48,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628685 2024-12-10T11:03:48,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628685 2024-12-10T11:03:48,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628685) roll requested 2024-12-10T11:03:48,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628692 2024-12-10T11:03:48,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628692 2024-12-10T11:03:48,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628692) roll requested 2024-12-10T11:03:48,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628698 2024-12-10T11:03:48,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628698 2024-12-10T11:03:48,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628698) roll requested 2024-12-10T11:03:48,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628704 2024-12-10T11:03:48,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628704 2024-12-10T11:03:48,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628704) roll requested 2024-12-10T11:03:48,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628710 2024-12-10T11:03:48,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628710 2024-12-10T11:03:48,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628710) roll requested 2024-12-10T11:03:48,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628716 2024-12-10T11:03:48,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628716 2024-12-10T11:03:48,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628716) roll requested 2024-12-10T11:03:48,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628723 2024-12-10T11:03:48,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628723 2024-12-10T11:03:48,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628723) roll requested 2024-12-10T11:03:48,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628729 2024-12-10T11:03:48,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628729 2024-12-10T11:03:48,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628729) roll requested 2024-12-10T11:03:48,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628736 2024-12-10T11:03:48,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628736 2024-12-10T11:03:48,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628736) roll requested 2024-12-10T11:03:48,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628742 2024-12-10T11:03:48,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628742 2024-12-10T11:03:48,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628742) roll requested 2024-12-10T11:03:48,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628749 2024-12-10T11:03:48,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628749 2024-12-10T11:03:48,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628749) roll requested 2024-12-10T11:03:48,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628755 2024-12-10T11:03:48,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628755 2024-12-10T11:03:48,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628755) roll requested 2024-12-10T11:03:48,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628762 2024-12-10T11:03:48,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628762 2024-12-10T11:03:48,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628762) roll requested 2024-12-10T11:03:48,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628768 2024-12-10T11:03:48,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628768 2024-12-10T11:03:48,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628768) roll requested 2024-12-10T11:03:48,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628775 2024-12-10T11:03:48,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628775 2024-12-10T11:03:48,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628775) roll requested 2024-12-10T11:03:48,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628781 2024-12-10T11:03:48,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628781 2024-12-10T11:03:48,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628781) roll requested 2024-12-10T11:03:48,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628787 2024-12-10T11:03:48,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628787 2024-12-10T11:03:48,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628787) roll requested 2024-12-10T11:03:48,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628794 2024-12-10T11:03:48,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628794 2024-12-10T11:03:48,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628794) roll requested 2024-12-10T11:03:48,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628800 2024-12-10T11:03:48,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628800 2024-12-10T11:03:48,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628800) roll requested 2024-12-10T11:03:48,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628806 2024-12-10T11:03:48,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628806 2024-12-10T11:03:48,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628806) roll requested 2024-12-10T11:03:48,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628812 2024-12-10T11:03:48,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628812 2024-12-10T11:03:48,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628812) roll requested 2024-12-10T11:03:48,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628819 2024-12-10T11:03:48,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628819 2024-12-10T11:03:48,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628819) roll requested 2024-12-10T11:03:48,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628826 2024-12-10T11:03:48,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628826 2024-12-10T11:03:48,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628826) roll requested 2024-12-10T11:03:48,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628838 2024-12-10T11:03:48,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628838 2024-12-10T11:03:48,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628838) roll requested 2024-12-10T11:03:48,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628844 2024-12-10T11:03:48,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628844 2024-12-10T11:03:48,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628844) roll requested 2024-12-10T11:03:48,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628850 2024-12-10T11:03:48,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628850 2024-12-10T11:03:48,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628850) roll requested 2024-12-10T11:03:48,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628856 2024-12-10T11:03:48,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628856 2024-12-10T11:03:48,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628856) roll requested 2024-12-10T11:03:48,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628862 2024-12-10T11:03:48,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628862 2024-12-10T11:03:48,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628862) roll requested 2024-12-10T11:03:48,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628869 2024-12-10T11:03:48,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628869 2024-12-10T11:03:48,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628869) roll requested 2024-12-10T11:03:48,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628875 2024-12-10T11:03:48,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628875 2024-12-10T11:03:48,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628875) roll requested 2024-12-10T11:03:48,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628881 2024-12-10T11:03:48,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628881 2024-12-10T11:03:48,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628881) roll requested 2024-12-10T11:03:48,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628887 2024-12-10T11:03:48,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628887 2024-12-10T11:03:48,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628887) roll requested 2024-12-10T11:03:48,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628894 2024-12-10T11:03:48,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628894 2024-12-10T11:03:48,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628894) roll requested 2024-12-10T11:03:48,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628905 2024-12-10T11:03:48,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628905 2024-12-10T11:03:48,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628905) roll requested 2024-12-10T11:03:48,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628912 2024-12-10T11:03:48,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628912 2024-12-10T11:03:48,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628912) roll requested 2024-12-10T11:03:48,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628918 2024-12-10T11:03:48,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628918 2024-12-10T11:03:48,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628918) roll requested 2024-12-10T11:03:48,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628926 2024-12-10T11:03:48,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628926 2024-12-10T11:03:48,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628926) roll requested 2024-12-10T11:03:48,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628932 2024-12-10T11:03:48,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628932 2024-12-10T11:03:48,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628932) roll requested 2024-12-10T11:03:48,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628939 2024-12-10T11:03:48,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628939 2024-12-10T11:03:48,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628939) roll requested 2024-12-10T11:03:48,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628945 2024-12-10T11:03:48,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628945 2024-12-10T11:03:48,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628945) roll requested 2024-12-10T11:03:48,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628952 2024-12-10T11:03:48,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628952 2024-12-10T11:03:48,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628952) roll requested 2024-12-10T11:03:48,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628958 2024-12-10T11:03:48,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628958 2024-12-10T11:03:48,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628958) roll requested 2024-12-10T11:03:48,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628964 2024-12-10T11:03:48,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628964 2024-12-10T11:03:48,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628964) roll requested 2024-12-10T11:03:48,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628970 2024-12-10T11:03:48,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628970 2024-12-10T11:03:48,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628970) roll requested 2024-12-10T11:03:48,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628977 2024-12-10T11:03:48,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628977 2024-12-10T11:03:48,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628977) roll requested 2024-12-10T11:03:48,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628983 2024-12-10T11:03:48,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628983 2024-12-10T11:03:48,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:48,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:48,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628983) roll requested 2024-12-10T11:03:48,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828628993 2024-12-10T11:03:48,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:48,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628993 2024-12-10T11:03:49,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828628993) roll requested 2024-12-10T11:03:49,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629000 2024-12-10T11:03:49,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828628993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629000 2024-12-10T11:03:49,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629000) roll requested 2024-12-10T11:03:49,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629006 2024-12-10T11:03:49,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629006 2024-12-10T11:03:49,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629006) roll requested 2024-12-10T11:03:49,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629012 2024-12-10T11:03:49,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629012 2024-12-10T11:03:49,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629012) roll requested 2024-12-10T11:03:49,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629018 2024-12-10T11:03:49,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629018 2024-12-10T11:03:49,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629018) roll requested 2024-12-10T11:03:49,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629024 2024-12-10T11:03:49,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629024 2024-12-10T11:03:49,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629024) roll requested 2024-12-10T11:03:49,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629031 2024-12-10T11:03:49,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629031 2024-12-10T11:03:49,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629031) roll requested 2024-12-10T11:03:49,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629037 2024-12-10T11:03:49,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629037 2024-12-10T11:03:49,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629037) roll requested 2024-12-10T11:03:49,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629044 2024-12-10T11:03:49,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629044 2024-12-10T11:03:49,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629044) roll requested 2024-12-10T11:03:49,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629050 2024-12-10T11:03:49,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629050 2024-12-10T11:03:49,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629050) roll requested 2024-12-10T11:03:49,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629057 2024-12-10T11:03:49,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629057 2024-12-10T11:03:49,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629057) roll requested 2024-12-10T11:03:49,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629063 2024-12-10T11:03:49,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629063 2024-12-10T11:03:49,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629063) roll requested 2024-12-10T11:03:49,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629070 2024-12-10T11:03:49,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629070 2024-12-10T11:03:49,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629070) roll requested 2024-12-10T11:03:49,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629076 2024-12-10T11:03:49,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629076 2024-12-10T11:03:49,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629076) roll requested 2024-12-10T11:03:49,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629082 2024-12-10T11:03:49,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629082 2024-12-10T11:03:49,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629082) roll requested 2024-12-10T11:03:49,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629088 2024-12-10T11:03:49,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629088 2024-12-10T11:03:49,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629088) roll requested 2024-12-10T11:03:49,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629095 2024-12-10T11:03:49,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629095 2024-12-10T11:03:49,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629095) roll requested 2024-12-10T11:03:49,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629101 2024-12-10T11:03:49,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629101 2024-12-10T11:03:49,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629101) roll requested 2024-12-10T11:03:49,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629107 2024-12-10T11:03:49,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629107 2024-12-10T11:03:49,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629107) roll requested 2024-12-10T11:03:49,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629114 2024-12-10T11:03:49,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629114 2024-12-10T11:03:49,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629114) roll requested 2024-12-10T11:03:49,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629120 2024-12-10T11:03:49,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629120 2024-12-10T11:03:49,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629120) roll requested 2024-12-10T11:03:49,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629126 2024-12-10T11:03:49,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629126 2024-12-10T11:03:49,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629126) roll requested 2024-12-10T11:03:49,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629133 2024-12-10T11:03:49,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629133 2024-12-10T11:03:49,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629133) roll requested 2024-12-10T11:03:49,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629139 2024-12-10T11:03:49,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629139 2024-12-10T11:03:49,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629139) roll requested 2024-12-10T11:03:49,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629145 2024-12-10T11:03:49,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629145 2024-12-10T11:03:49,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629145) roll requested 2024-12-10T11:03:49,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629152 2024-12-10T11:03:49,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629152 2024-12-10T11:03:49,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629152) roll requested 2024-12-10T11:03:49,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629158 2024-12-10T11:03:49,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629158 2024-12-10T11:03:49,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629158) roll requested 2024-12-10T11:03:49,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629164 2024-12-10T11:03:49,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629164 2024-12-10T11:03:49,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629164) roll requested 2024-12-10T11:03:49,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629171 2024-12-10T11:03:49,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629171 2024-12-10T11:03:49,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629171) roll requested 2024-12-10T11:03:49,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629177 2024-12-10T11:03:49,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629177 2024-12-10T11:03:49,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629177) roll requested 2024-12-10T11:03:49,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629183 2024-12-10T11:03:49,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629183 2024-12-10T11:03:49,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629183) roll requested 2024-12-10T11:03:49,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629190 2024-12-10T11:03:49,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629190 2024-12-10T11:03:49,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629190) roll requested 2024-12-10T11:03:49,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629196 2024-12-10T11:03:49,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629196 2024-12-10T11:03:49,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629196) roll requested 2024-12-10T11:03:49,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629202 2024-12-10T11:03:49,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629202 2024-12-10T11:03:49,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629202) roll requested 2024-12-10T11:03:49,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629208 2024-12-10T11:03:49,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629208 2024-12-10T11:03:49,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629208) roll requested 2024-12-10T11:03:49,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629214 2024-12-10T11:03:49,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629214 2024-12-10T11:03:49,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629214) roll requested 2024-12-10T11:03:49,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629220 2024-12-10T11:03:49,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629220 2024-12-10T11:03:49,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629220) roll requested 2024-12-10T11:03:49,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629226 2024-12-10T11:03:49,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629226 2024-12-10T11:03:49,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629226) roll requested 2024-12-10T11:03:49,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629233 2024-12-10T11:03:49,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629233 2024-12-10T11:03:49,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629233) roll requested 2024-12-10T11:03:49,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629239 2024-12-10T11:03:49,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629239 2024-12-10T11:03:49,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629239) roll requested 2024-12-10T11:03:49,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629245 2024-12-10T11:03:49,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629245 2024-12-10T11:03:49,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629245) roll requested 2024-12-10T11:03:49,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629251 2024-12-10T11:03:49,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629251 2024-12-10T11:03:49,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629251) roll requested 2024-12-10T11:03:49,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629258 2024-12-10T11:03:49,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629258 2024-12-10T11:03:49,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629258) roll requested 2024-12-10T11:03:49,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629264 2024-12-10T11:03:49,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629264 2024-12-10T11:03:49,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629264) roll requested 2024-12-10T11:03:49,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629270 2024-12-10T11:03:49,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629270 2024-12-10T11:03:49,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629270) roll requested 2024-12-10T11:03:49,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629276 2024-12-10T11:03:49,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629276 2024-12-10T11:03:49,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629276) roll requested 2024-12-10T11:03:49,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629282 2024-12-10T11:03:49,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629282 2024-12-10T11:03:49,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629282) roll requested 2024-12-10T11:03:49,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629289 2024-12-10T11:03:49,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629289 2024-12-10T11:03:49,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629289) roll requested 2024-12-10T11:03:49,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629295 2024-12-10T11:03:49,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629295 2024-12-10T11:03:49,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629295) roll requested 2024-12-10T11:03:49,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629301 2024-12-10T11:03:49,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629301 2024-12-10T11:03:49,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629301) roll requested 2024-12-10T11:03:49,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629307 2024-12-10T11:03:49,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629307 2024-12-10T11:03:49,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629307) roll requested 2024-12-10T11:03:49,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629314 2024-12-10T11:03:49,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629314 2024-12-10T11:03:49,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629314) roll requested 2024-12-10T11:03:49,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629320 2024-12-10T11:03:49,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629320 2024-12-10T11:03:49,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629320) roll requested 2024-12-10T11:03:49,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629326 2024-12-10T11:03:49,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629326 2024-12-10T11:03:49,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629326) roll requested 2024-12-10T11:03:49,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629333 2024-12-10T11:03:49,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629333 2024-12-10T11:03:49,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629333) roll requested 2024-12-10T11:03:49,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629341 2024-12-10T11:03:49,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629341 2024-12-10T11:03:49,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629341) roll requested 2024-12-10T11:03:49,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629348 2024-12-10T11:03:49,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629348 2024-12-10T11:03:49,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629348) roll requested 2024-12-10T11:03:49,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629354 2024-12-10T11:03:49,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629354 2024-12-10T11:03:49,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629354) roll requested 2024-12-10T11:03:49,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629361 2024-12-10T11:03:49,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629361 2024-12-10T11:03:49,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629361) roll requested 2024-12-10T11:03:49,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629367 2024-12-10T11:03:49,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629367 2024-12-10T11:03:49,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629367) roll requested 2024-12-10T11:03:49,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629373 2024-12-10T11:03:49,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629373 2024-12-10T11:03:49,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629373) roll requested 2024-12-10T11:03:49,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629380 2024-12-10T11:03:49,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629380 2024-12-10T11:03:49,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629380) roll requested 2024-12-10T11:03:49,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629386 2024-12-10T11:03:49,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629386 2024-12-10T11:03:49,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629386) roll requested 2024-12-10T11:03:49,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629393 2024-12-10T11:03:49,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629393 2024-12-10T11:03:49,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629393) roll requested 2024-12-10T11:03:49,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629399 2024-12-10T11:03:49,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629399 2024-12-10T11:03:49,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629399) roll requested 2024-12-10T11:03:49,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629405 2024-12-10T11:03:49,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629405 2024-12-10T11:03:49,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629405) roll requested 2024-12-10T11:03:49,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629411 2024-12-10T11:03:49,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629411 2024-12-10T11:03:49,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629411) roll requested 2024-12-10T11:03:49,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629417 2024-12-10T11:03:49,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629417 2024-12-10T11:03:49,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629417) roll requested 2024-12-10T11:03:49,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629426 2024-12-10T11:03:49,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629426 2024-12-10T11:03:49,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629426) roll requested 2024-12-10T11:03:49,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629438 2024-12-10T11:03:49,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629438 2024-12-10T11:03:49,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629438) roll requested 2024-12-10T11:03:49,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629445 2024-12-10T11:03:49,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629445 2024-12-10T11:03:49,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629445) roll requested 2024-12-10T11:03:49,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629451 2024-12-10T11:03:49,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629451 2024-12-10T11:03:49,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629451) roll requested 2024-12-10T11:03:49,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629457 2024-12-10T11:03:49,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629457 2024-12-10T11:03:49,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629457) roll requested 2024-12-10T11:03:49,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629463 2024-12-10T11:03:49,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629463 2024-12-10T11:03:49,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629463) roll requested 2024-12-10T11:03:49,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629470 2024-12-10T11:03:49,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629470 2024-12-10T11:03:49,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629470) roll requested 2024-12-10T11:03:49,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629476 2024-12-10T11:03:49,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629476 2024-12-10T11:03:49,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629476) roll requested 2024-12-10T11:03:49,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629482 2024-12-10T11:03:49,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629482 2024-12-10T11:03:49,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629482) roll requested 2024-12-10T11:03:49,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629488 2024-12-10T11:03:49,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629488 2024-12-10T11:03:49,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629488) roll requested 2024-12-10T11:03:49,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629497 2024-12-10T11:03:49,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629497 2024-12-10T11:03:49,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629497) roll requested 2024-12-10T11:03:49,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629505 2024-12-10T11:03:49,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629505 2024-12-10T11:03:49,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629505) roll requested 2024-12-10T11:03:49,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629511 2024-12-10T11:03:49,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629511 2024-12-10T11:03:49,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629511) roll requested 2024-12-10T11:03:49,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629517 2024-12-10T11:03:49,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629517 2024-12-10T11:03:49,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629517) roll requested 2024-12-10T11:03:49,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629528 2024-12-10T11:03:49,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629528 2024-12-10T11:03:49,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629528) roll requested 2024-12-10T11:03:49,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629534 2024-12-10T11:03:49,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629534 2024-12-10T11:03:49,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629534) roll requested 2024-12-10T11:03:49,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629541 2024-12-10T11:03:49,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629541 2024-12-10T11:03:49,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629541) roll requested 2024-12-10T11:03:49,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629548 2024-12-10T11:03:49,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629548 2024-12-10T11:03:49,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629548) roll requested 2024-12-10T11:03:49,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629555 2024-12-10T11:03:49,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629555 2024-12-10T11:03:49,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629555) roll requested 2024-12-10T11:03:49,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629561 2024-12-10T11:03:49,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629561 2024-12-10T11:03:49,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629561) roll requested 2024-12-10T11:03:49,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629567 2024-12-10T11:03:49,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629567 2024-12-10T11:03:49,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629567) roll requested 2024-12-10T11:03:49,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629575 2024-12-10T11:03:49,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629575 2024-12-10T11:03:49,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629575) roll requested 2024-12-10T11:03:49,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629581 2024-12-10T11:03:49,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629581 2024-12-10T11:03:49,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629581) roll requested 2024-12-10T11:03:49,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629587 2024-12-10T11:03:49,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629587 2024-12-10T11:03:49,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629587) roll requested 2024-12-10T11:03:49,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629594 2024-12-10T11:03:49,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629594 2024-12-10T11:03:49,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629594) roll requested 2024-12-10T11:03:49,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629604 2024-12-10T11:03:49,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629604 2024-12-10T11:03:49,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629604) roll requested 2024-12-10T11:03:49,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629610 2024-12-10T11:03:49,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629610 2024-12-10T11:03:49,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629610) roll requested 2024-12-10T11:03:49,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629617 2024-12-10T11:03:49,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629617 2024-12-10T11:03:49,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629617) roll requested 2024-12-10T11:03:49,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629623 2024-12-10T11:03:49,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629623 2024-12-10T11:03:49,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629623) roll requested 2024-12-10T11:03:49,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629630 2024-12-10T11:03:49,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629630 2024-12-10T11:03:49,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629630) roll requested 2024-12-10T11:03:49,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629636 2024-12-10T11:03:49,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629636 2024-12-10T11:03:49,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629636) roll requested 2024-12-10T11:03:49,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629642 2024-12-10T11:03:49,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629642 2024-12-10T11:03:49,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629642) roll requested 2024-12-10T11:03:49,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629648 2024-12-10T11:03:49,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629648 2024-12-10T11:03:49,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629648) roll requested 2024-12-10T11:03:49,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629654 2024-12-10T11:03:49,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629654 2024-12-10T11:03:49,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629654) roll requested 2024-12-10T11:03:49,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629661 2024-12-10T11:03:49,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629661 2024-12-10T11:03:49,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629661) roll requested 2024-12-10T11:03:49,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629667 2024-12-10T11:03:49,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629667 2024-12-10T11:03:49,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629667) roll requested 2024-12-10T11:03:49,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629673 2024-12-10T11:03:49,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629673 2024-12-10T11:03:49,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629673) roll requested 2024-12-10T11:03:49,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629679 2024-12-10T11:03:49,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629679 2024-12-10T11:03:49,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629679) roll requested 2024-12-10T11:03:49,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629688 2024-12-10T11:03:49,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629688 2024-12-10T11:03:49,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629688) roll requested 2024-12-10T11:03:49,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629695 2024-12-10T11:03:49,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629695 2024-12-10T11:03:49,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629695) roll requested 2024-12-10T11:03:49,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629701 2024-12-10T11:03:49,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629701 2024-12-10T11:03:49,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629701) roll requested 2024-12-10T11:03:49,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629709 2024-12-10T11:03:49,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629709 2024-12-10T11:03:49,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629709) roll requested 2024-12-10T11:03:49,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629715 2024-12-10T11:03:49,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629715 2024-12-10T11:03:49,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629715) roll requested 2024-12-10T11:03:49,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629722 2024-12-10T11:03:49,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629722 2024-12-10T11:03:49,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629722) roll requested 2024-12-10T11:03:49,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629728 2024-12-10T11:03:49,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629728 2024-12-10T11:03:49,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629728) roll requested 2024-12-10T11:03:49,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629735 2024-12-10T11:03:49,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629735 2024-12-10T11:03:49,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629735) roll requested 2024-12-10T11:03:49,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629741 2024-12-10T11:03:49,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629741 2024-12-10T11:03:49,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629741) roll requested 2024-12-10T11:03:49,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629747 2024-12-10T11:03:49,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629747 2024-12-10T11:03:49,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629747) roll requested 2024-12-10T11:03:49,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629753 2024-12-10T11:03:49,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629753 2024-12-10T11:03:49,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629753) roll requested 2024-12-10T11:03:49,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629759 2024-12-10T11:03:49,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629759 2024-12-10T11:03:49,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629759) roll requested 2024-12-10T11:03:49,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629766 2024-12-10T11:03:49,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629766 2024-12-10T11:03:49,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629766) roll requested 2024-12-10T11:03:49,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629775 2024-12-10T11:03:49,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629775 2024-12-10T11:03:49,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629775) roll requested 2024-12-10T11:03:49,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629781 2024-12-10T11:03:49,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629781 2024-12-10T11:03:49,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629781) roll requested 2024-12-10T11:03:49,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629787 2024-12-10T11:03:49,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629787 2024-12-10T11:03:49,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629787) roll requested 2024-12-10T11:03:49,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629794 2024-12-10T11:03:49,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629794 2024-12-10T11:03:49,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629794) roll requested 2024-12-10T11:03:49,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629800 2024-12-10T11:03:49,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629800 2024-12-10T11:03:49,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629800) roll requested 2024-12-10T11:03:49,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629806 2024-12-10T11:03:49,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629806 2024-12-10T11:03:49,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629806) roll requested 2024-12-10T11:03:49,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629812 2024-12-10T11:03:49,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629812 2024-12-10T11:03:49,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629812) roll requested 2024-12-10T11:03:49,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629819 2024-12-10T11:03:49,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629819 2024-12-10T11:03:49,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629819) roll requested 2024-12-10T11:03:49,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629825 2024-12-10T11:03:49,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629825 2024-12-10T11:03:49,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629825) roll requested 2024-12-10T11:03:49,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629831 2024-12-10T11:03:49,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629831 2024-12-10T11:03:49,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629831) roll requested 2024-12-10T11:03:49,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629837 2024-12-10T11:03:49,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629837 2024-12-10T11:03:49,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629837) roll requested 2024-12-10T11:03:49,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629843 2024-12-10T11:03:49,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629843 2024-12-10T11:03:49,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629843) roll requested 2024-12-10T11:03:49,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629850 2024-12-10T11:03:49,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629850 2024-12-10T11:03:49,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629850) roll requested 2024-12-10T11:03:49,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629856 2024-12-10T11:03:49,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629856 2024-12-10T11:03:49,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629856) roll requested 2024-12-10T11:03:49,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629862 2024-12-10T11:03:49,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629862 2024-12-10T11:03:49,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629862) roll requested 2024-12-10T11:03:49,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629868 2024-12-10T11:03:49,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629868 2024-12-10T11:03:49,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629868) roll requested 2024-12-10T11:03:49,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629875 2024-12-10T11:03:49,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629875 2024-12-10T11:03:49,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629875) roll requested 2024-12-10T11:03:49,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629881 2024-12-10T11:03:49,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629881 2024-12-10T11:03:49,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629881) roll requested 2024-12-10T11:03:49,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629887 2024-12-10T11:03:49,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629887 2024-12-10T11:03:49,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629887) roll requested 2024-12-10T11:03:49,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629894 2024-12-10T11:03:49,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629894 2024-12-10T11:03:49,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629894) roll requested 2024-12-10T11:03:49,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629900 2024-12-10T11:03:49,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629900 2024-12-10T11:03:49,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629900) roll requested 2024-12-10T11:03:49,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629907 2024-12-10T11:03:49,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629907 2024-12-10T11:03:49,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629907) roll requested 2024-12-10T11:03:49,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629913 2024-12-10T11:03:49,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629913 2024-12-10T11:03:49,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629913) roll requested 2024-12-10T11:03:49,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629919 2024-12-10T11:03:49,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629919 2024-12-10T11:03:49,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629919) roll requested 2024-12-10T11:03:49,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629925 2024-12-10T11:03:49,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629925 2024-12-10T11:03:49,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629925) roll requested 2024-12-10T11:03:49,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629932 2024-12-10T11:03:49,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629932 2024-12-10T11:03:49,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629932) roll requested 2024-12-10T11:03:49,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629938 2024-12-10T11:03:49,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629938 2024-12-10T11:03:49,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629938) roll requested 2024-12-10T11:03:49,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629944 2024-12-10T11:03:49,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629944 2024-12-10T11:03:49,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629944) roll requested 2024-12-10T11:03:49,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629950 2024-12-10T11:03:49,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629950 2024-12-10T11:03:49,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629950) roll requested 2024-12-10T11:03:49,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629957 2024-12-10T11:03:49,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629957 2024-12-10T11:03:49,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629957) roll requested 2024-12-10T11:03:49,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629963 2024-12-10T11:03:49,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629963 2024-12-10T11:03:49,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629963) roll requested 2024-12-10T11:03:49,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629969 2024-12-10T11:03:49,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629969 2024-12-10T11:03:49,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629969) roll requested 2024-12-10T11:03:49,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629975 2024-12-10T11:03:49,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629975 2024-12-10T11:03:49,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629975) roll requested 2024-12-10T11:03:49,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629981 2024-12-10T11:03:49,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629981 2024-12-10T11:03:49,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629981) roll requested 2024-12-10T11:03:49,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629988 2024-12-10T11:03:49,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:49,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629988 2024-12-10T11:03:49,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:49,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:49,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629988) roll requested 2024-12-10T11:03:49,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828629997 2024-12-10T11:03:50,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629997 2024-12-10T11:03:50,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828629997) roll requested 2024-12-10T11:03:50,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630004 2024-12-10T11:03:50,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828629997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630004 2024-12-10T11:03:50,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630004) roll requested 2024-12-10T11:03:50,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630010 2024-12-10T11:03:50,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630010 2024-12-10T11:03:50,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630010) roll requested 2024-12-10T11:03:50,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630016 2024-12-10T11:03:50,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630016 2024-12-10T11:03:50,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630016) roll requested 2024-12-10T11:03:50,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630023 2024-12-10T11:03:50,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630023 2024-12-10T11:03:50,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630023) roll requested 2024-12-10T11:03:50,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630034 2024-12-10T11:03:50,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630034 2024-12-10T11:03:50,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630034) roll requested 2024-12-10T11:03:50,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630040 2024-12-10T11:03:50,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630040 2024-12-10T11:03:50,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630040) roll requested 2024-12-10T11:03:50,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630047 2024-12-10T11:03:50,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630047 2024-12-10T11:03:50,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630047) roll requested 2024-12-10T11:03:50,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630053 2024-12-10T11:03:50,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630053 2024-12-10T11:03:50,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630053) roll requested 2024-12-10T11:03:50,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630059 2024-12-10T11:03:50,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630059 2024-12-10T11:03:50,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630059) roll requested 2024-12-10T11:03:50,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630065 2024-12-10T11:03:50,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630065 2024-12-10T11:03:50,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630065) roll requested 2024-12-10T11:03:50,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630072 2024-12-10T11:03:50,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630072 2024-12-10T11:03:50,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630072) roll requested 2024-12-10T11:03:50,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630078 2024-12-10T11:03:50,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630078 2024-12-10T11:03:50,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630078) roll requested 2024-12-10T11:03:50,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630084 2024-12-10T11:03:50,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630084 2024-12-10T11:03:50,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630084) roll requested 2024-12-10T11:03:50,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630091 2024-12-10T11:03:50,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630091 2024-12-10T11:03:50,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630091) roll requested 2024-12-10T11:03:50,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630097 2024-12-10T11:03:50,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630097 2024-12-10T11:03:50,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630097) roll requested 2024-12-10T11:03:50,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630104 2024-12-10T11:03:50,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630104 2024-12-10T11:03:50,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630104) roll requested 2024-12-10T11:03:50,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630110 2024-12-10T11:03:50,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630110 2024-12-10T11:03:50,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630110) roll requested 2024-12-10T11:03:50,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630116 2024-12-10T11:03:50,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630116 2024-12-10T11:03:50,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630116) roll requested 2024-12-10T11:03:50,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630122 2024-12-10T11:03:50,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630122 2024-12-10T11:03:50,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630122) roll requested 2024-12-10T11:03:50,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630128 2024-12-10T11:03:50,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630128 2024-12-10T11:03:50,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630128) roll requested 2024-12-10T11:03:50,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630135 2024-12-10T11:03:50,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630135 2024-12-10T11:03:50,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630135) roll requested 2024-12-10T11:03:50,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630141 2024-12-10T11:03:50,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630141 2024-12-10T11:03:50,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630141) roll requested 2024-12-10T11:03:50,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630147 2024-12-10T11:03:50,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630147 2024-12-10T11:03:50,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630147) roll requested 2024-12-10T11:03:50,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630153 2024-12-10T11:03:50,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630153 2024-12-10T11:03:50,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630153) roll requested 2024-12-10T11:03:50,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630159 2024-12-10T11:03:50,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630159 2024-12-10T11:03:50,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630159) roll requested 2024-12-10T11:03:50,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630166 2024-12-10T11:03:50,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630166 2024-12-10T11:03:50,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630166) roll requested 2024-12-10T11:03:50,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630172 2024-12-10T11:03:50,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630172 2024-12-10T11:03:50,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630172) roll requested 2024-12-10T11:03:50,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630178 2024-12-10T11:03:50,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630178 2024-12-10T11:03:50,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630178) roll requested 2024-12-10T11:03:50,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630185 2024-12-10T11:03:50,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630185 2024-12-10T11:03:50,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630185) roll requested 2024-12-10T11:03:50,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630192 2024-12-10T11:03:50,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630192 2024-12-10T11:03:50,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630192) roll requested 2024-12-10T11:03:50,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630198 2024-12-10T11:03:50,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630198 2024-12-10T11:03:50,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630198) roll requested 2024-12-10T11:03:50,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630205 2024-12-10T11:03:50,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630205 2024-12-10T11:03:50,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630205) roll requested 2024-12-10T11:03:50,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630211 2024-12-10T11:03:50,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630211 2024-12-10T11:03:50,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630211) roll requested 2024-12-10T11:03:50,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630218 2024-12-10T11:03:50,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630218 2024-12-10T11:03:50,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630218) roll requested 2024-12-10T11:03:50,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630224 2024-12-10T11:03:50,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630224 2024-12-10T11:03:50,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630224) roll requested 2024-12-10T11:03:50,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630230 2024-12-10T11:03:50,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630230 2024-12-10T11:03:50,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630230) roll requested 2024-12-10T11:03:50,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630237 2024-12-10T11:03:50,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630237 2024-12-10T11:03:50,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630237) roll requested 2024-12-10T11:03:50,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630243 2024-12-10T11:03:50,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630243 2024-12-10T11:03:50,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630243) roll requested 2024-12-10T11:03:50,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630249 2024-12-10T11:03:50,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630249 2024-12-10T11:03:50,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630249) roll requested 2024-12-10T11:03:50,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630255 2024-12-10T11:03:50,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630255 2024-12-10T11:03:50,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630255) roll requested 2024-12-10T11:03:50,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630262 2024-12-10T11:03:50,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630262 2024-12-10T11:03:50,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630262) roll requested 2024-12-10T11:03:50,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630268 2024-12-10T11:03:50,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630268 2024-12-10T11:03:50,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630268) roll requested 2024-12-10T11:03:50,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630274 2024-12-10T11:03:50,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630274 2024-12-10T11:03:50,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630274) roll requested 2024-12-10T11:03:50,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630281 2024-12-10T11:03:50,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630281 2024-12-10T11:03:50,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630281) roll requested 2024-12-10T11:03:50,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630287 2024-12-10T11:03:50,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630287 2024-12-10T11:03:50,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630287) roll requested 2024-12-10T11:03:50,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630293 2024-12-10T11:03:50,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630293 2024-12-10T11:03:50,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630293) roll requested 2024-12-10T11:03:50,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630299 2024-12-10T11:03:50,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630299 2024-12-10T11:03:50,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630299) roll requested 2024-12-10T11:03:50,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630306 2024-12-10T11:03:50,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630306 2024-12-10T11:03:50,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630306) roll requested 2024-12-10T11:03:50,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630312 2024-12-10T11:03:50,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630312 2024-12-10T11:03:50,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630312) roll requested 2024-12-10T11:03:50,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630318 2024-12-10T11:03:50,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630318 2024-12-10T11:03:50,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630318) roll requested 2024-12-10T11:03:50,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630325 2024-12-10T11:03:50,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630325 2024-12-10T11:03:50,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630325) roll requested 2024-12-10T11:03:50,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630331 2024-12-10T11:03:50,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630331 2024-12-10T11:03:50,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630331) roll requested 2024-12-10T11:03:50,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630339 2024-12-10T11:03:50,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630339 2024-12-10T11:03:50,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630339) roll requested 2024-12-10T11:03:50,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630346 2024-12-10T11:03:50,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630346 2024-12-10T11:03:50,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630346) roll requested 2024-12-10T11:03:50,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630353 2024-12-10T11:03:50,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630353 2024-12-10T11:03:50,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630353) roll requested 2024-12-10T11:03:50,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630359 2024-12-10T11:03:50,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630359 2024-12-10T11:03:50,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630359) roll requested 2024-12-10T11:03:50,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630366 2024-12-10T11:03:50,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630366 2024-12-10T11:03:50,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630366) roll requested 2024-12-10T11:03:50,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630372 2024-12-10T11:03:50,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630372 2024-12-10T11:03:50,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630372) roll requested 2024-12-10T11:03:50,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630378 2024-12-10T11:03:50,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630378 2024-12-10T11:03:50,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630378) roll requested 2024-12-10T11:03:50,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630384 2024-12-10T11:03:50,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630384 2024-12-10T11:03:50,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630384) roll requested 2024-12-10T11:03:50,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630394 2024-12-10T11:03:50,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630394 2024-12-10T11:03:50,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630394) roll requested 2024-12-10T11:03:50,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630401 2024-12-10T11:03:50,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630401 2024-12-10T11:03:50,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630401) roll requested 2024-12-10T11:03:50,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630413 2024-12-10T11:03:50,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630413 2024-12-10T11:03:50,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630413) roll requested 2024-12-10T11:03:50,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630420 2024-12-10T11:03:50,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630420 2024-12-10T11:03:50,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630420) roll requested 2024-12-10T11:03:50,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630428 2024-12-10T11:03:50,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630428 2024-12-10T11:03:50,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630428) roll requested 2024-12-10T11:03:50,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630435 2024-12-10T11:03:50,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630435 2024-12-10T11:03:50,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630435) roll requested 2024-12-10T11:03:50,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630441 2024-12-10T11:03:50,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630441 2024-12-10T11:03:50,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630441) roll requested 2024-12-10T11:03:50,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630447 2024-12-10T11:03:50,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630447 2024-12-10T11:03:50,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630447) roll requested 2024-12-10T11:03:50,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630454 2024-12-10T11:03:50,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630454 2024-12-10T11:03:50,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630454) roll requested 2024-12-10T11:03:50,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630461 2024-12-10T11:03:50,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630461 2024-12-10T11:03:50,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630461) roll requested 2024-12-10T11:03:50,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630473 2024-12-10T11:03:50,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630473 2024-12-10T11:03:50,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630473) roll requested 2024-12-10T11:03:50,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630480 2024-12-10T11:03:50,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630480 2024-12-10T11:03:50,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630480) roll requested 2024-12-10T11:03:50,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630493 2024-12-10T11:03:50,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630493 2024-12-10T11:03:50,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630493) roll requested 2024-12-10T11:03:50,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630501 2024-12-10T11:03:50,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630501 2024-12-10T11:03:50,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630501) roll requested 2024-12-10T11:03:50,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630508 2024-12-10T11:03:50,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630508 2024-12-10T11:03:50,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630508) roll requested 2024-12-10T11:03:50,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630517 2024-12-10T11:03:50,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630517 2024-12-10T11:03:50,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630517) roll requested 2024-12-10T11:03:50,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630529 2024-12-10T11:03:50,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630529 2024-12-10T11:03:50,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630529) roll requested 2024-12-10T11:03:50,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630536 2024-12-10T11:03:50,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630536 2024-12-10T11:03:50,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630536) roll requested 2024-12-10T11:03:50,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630542 2024-12-10T11:03:50,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630542 2024-12-10T11:03:50,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630542) roll requested 2024-12-10T11:03:50,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630549 2024-12-10T11:03:50,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630549 2024-12-10T11:03:50,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630549) roll requested 2024-12-10T11:03:50,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630555 2024-12-10T11:03:50,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630555 2024-12-10T11:03:50,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630555) roll requested 2024-12-10T11:03:50,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630561 2024-12-10T11:03:50,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630561 2024-12-10T11:03:50,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630561) roll requested 2024-12-10T11:03:50,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630567 2024-12-10T11:03:50,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630567 2024-12-10T11:03:50,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630567) roll requested 2024-12-10T11:03:50,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630574 2024-12-10T11:03:50,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630574 2024-12-10T11:03:50,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630574) roll requested 2024-12-10T11:03:50,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630580 2024-12-10T11:03:50,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630580 2024-12-10T11:03:50,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630580) roll requested 2024-12-10T11:03:50,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630587 2024-12-10T11:03:50,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630587 2024-12-10T11:03:50,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630587) roll requested 2024-12-10T11:03:50,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630594 2024-12-10T11:03:50,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630594 2024-12-10T11:03:50,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630594) roll requested 2024-12-10T11:03:50,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630601 2024-12-10T11:03:50,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630601 2024-12-10T11:03:50,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630601) roll requested 2024-12-10T11:03:50,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630608 2024-12-10T11:03:50,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630608 2024-12-10T11:03:50,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630608) roll requested 2024-12-10T11:03:50,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630614 2024-12-10T11:03:50,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630614 2024-12-10T11:03:50,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630614) roll requested 2024-12-10T11:03:50,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630620 2024-12-10T11:03:50,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630620 2024-12-10T11:03:50,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630620) roll requested 2024-12-10T11:03:50,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630627 2024-12-10T11:03:50,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630627 2024-12-10T11:03:50,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630627) roll requested 2024-12-10T11:03:50,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630633 2024-12-10T11:03:50,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630633 2024-12-10T11:03:50,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630633) roll requested 2024-12-10T11:03:50,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630644 2024-12-10T11:03:50,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630644 2024-12-10T11:03:50,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630644) roll requested 2024-12-10T11:03:50,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630651 2024-12-10T11:03:50,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630651 2024-12-10T11:03:50,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630651) roll requested 2024-12-10T11:03:50,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630662 2024-12-10T11:03:50,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630662 2024-12-10T11:03:50,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630662) roll requested 2024-12-10T11:03:50,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630668 2024-12-10T11:03:50,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630668 2024-12-10T11:03:50,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630668) roll requested 2024-12-10T11:03:50,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630676 2024-12-10T11:03:50,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630676 2024-12-10T11:03:50,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630676) roll requested 2024-12-10T11:03:50,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630682 2024-12-10T11:03:50,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630682 2024-12-10T11:03:50,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630682) roll requested 2024-12-10T11:03:50,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630689 2024-12-10T11:03:50,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630689 2024-12-10T11:03:50,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630689) roll requested 2024-12-10T11:03:50,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630696 2024-12-10T11:03:50,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630696 2024-12-10T11:03:50,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630696) roll requested 2024-12-10T11:03:50,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630702 2024-12-10T11:03:50,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630702 2024-12-10T11:03:50,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630702) roll requested 2024-12-10T11:03:50,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630709 2024-12-10T11:03:50,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630709 2024-12-10T11:03:50,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630709) roll requested 2024-12-10T11:03:50,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630715 2024-12-10T11:03:50,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630715 2024-12-10T11:03:50,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630715) roll requested 2024-12-10T11:03:50,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630721 2024-12-10T11:03:50,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630721 2024-12-10T11:03:50,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630721) roll requested 2024-12-10T11:03:50,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630727 2024-12-10T11:03:50,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630727 2024-12-10T11:03:50,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630727) roll requested 2024-12-10T11:03:50,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630734 2024-12-10T11:03:50,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630734 2024-12-10T11:03:50,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630734) roll requested 2024-12-10T11:03:50,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630740 2024-12-10T11:03:50,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630740 2024-12-10T11:03:50,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630740) roll requested 2024-12-10T11:03:50,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630746 2024-12-10T11:03:50,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630746 2024-12-10T11:03:50,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630746) roll requested 2024-12-10T11:03:50,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630753 2024-12-10T11:03:50,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630753 2024-12-10T11:03:50,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630753) roll requested 2024-12-10T11:03:50,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630759 2024-12-10T11:03:50,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630759 2024-12-10T11:03:50,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630759) roll requested 2024-12-10T11:03:50,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630766 2024-12-10T11:03:50,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630766 2024-12-10T11:03:50,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630766) roll requested 2024-12-10T11:03:50,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630772 2024-12-10T11:03:50,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630772 2024-12-10T11:03:50,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630772) roll requested 2024-12-10T11:03:50,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630779 2024-12-10T11:03:50,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630779 2024-12-10T11:03:50,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630779) roll requested 2024-12-10T11:03:50,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630786 2024-12-10T11:03:50,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630786 2024-12-10T11:03:50,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630786) roll requested 2024-12-10T11:03:50,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630793 2024-12-10T11:03:50,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630793 2024-12-10T11:03:50,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630793) roll requested 2024-12-10T11:03:50,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630799 2024-12-10T11:03:50,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630799 2024-12-10T11:03:50,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630799) roll requested 2024-12-10T11:03:50,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630805 2024-12-10T11:03:50,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630805 2024-12-10T11:03:50,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630805) roll requested 2024-12-10T11:03:50,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630811 2024-12-10T11:03:50,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630811 2024-12-10T11:03:50,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630811) roll requested 2024-12-10T11:03:50,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630817 2024-12-10T11:03:50,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630817 2024-12-10T11:03:50,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630817) roll requested 2024-12-10T11:03:50,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630824 2024-12-10T11:03:50,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630824 2024-12-10T11:03:50,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630824) roll requested 2024-12-10T11:03:50,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630830 2024-12-10T11:03:50,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630830 2024-12-10T11:03:50,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630830) roll requested 2024-12-10T11:03:50,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630837 2024-12-10T11:03:50,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630837 2024-12-10T11:03:50,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630837) roll requested 2024-12-10T11:03:50,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630843 2024-12-10T11:03:50,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630843 2024-12-10T11:03:50,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630843) roll requested 2024-12-10T11:03:50,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630849 2024-12-10T11:03:50,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630849 2024-12-10T11:03:50,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630849) roll requested 2024-12-10T11:03:50,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630855 2024-12-10T11:03:50,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630855 2024-12-10T11:03:50,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630855) roll requested 2024-12-10T11:03:50,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630861 2024-12-10T11:03:50,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630861 2024-12-10T11:03:50,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630861) roll requested 2024-12-10T11:03:50,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630868 2024-12-10T11:03:50,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630868 2024-12-10T11:03:50,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630868) roll requested 2024-12-10T11:03:50,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630874 2024-12-10T11:03:50,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630874 2024-12-10T11:03:50,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630874) roll requested 2024-12-10T11:03:50,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630881 2024-12-10T11:03:50,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630881 2024-12-10T11:03:50,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630881) roll requested 2024-12-10T11:03:50,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630887 2024-12-10T11:03:50,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630887 2024-12-10T11:03:50,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630887) roll requested 2024-12-10T11:03:50,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630893 2024-12-10T11:03:50,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630893 2024-12-10T11:03:50,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630893) roll requested 2024-12-10T11:03:50,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630899 2024-12-10T11:03:50,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630899 2024-12-10T11:03:50,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630899) roll requested 2024-12-10T11:03:50,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630906 2024-12-10T11:03:50,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630906 2024-12-10T11:03:50,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630906) roll requested 2024-12-10T11:03:50,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630912 2024-12-10T11:03:50,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630912 2024-12-10T11:03:50,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630912) roll requested 2024-12-10T11:03:50,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630918 2024-12-10T11:03:50,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630918 2024-12-10T11:03:50,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630918) roll requested 2024-12-10T11:03:50,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630930 2024-12-10T11:03:50,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630930 2024-12-10T11:03:50,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630930) roll requested 2024-12-10T11:03:50,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630936 2024-12-10T11:03:50,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630936 2024-12-10T11:03:50,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630936) roll requested 2024-12-10T11:03:50,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630943 2024-12-10T11:03:50,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630943 2024-12-10T11:03:50,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630943) roll requested 2024-12-10T11:03:50,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630949 2024-12-10T11:03:50,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630949 2024-12-10T11:03:50,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630949) roll requested 2024-12-10T11:03:50,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630956 2024-12-10T11:03:50,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630956 2024-12-10T11:03:50,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630956) roll requested 2024-12-10T11:03:50,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630962 2024-12-10T11:03:50,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630962 2024-12-10T11:03:50,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630962) roll requested 2024-12-10T11:03:50,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630969 2024-12-10T11:03:50,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630969 2024-12-10T11:03:50,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630969) roll requested 2024-12-10T11:03:50,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630976 2024-12-10T11:03:50,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630976 2024-12-10T11:03:50,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630976) roll requested 2024-12-10T11:03:50,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630983 2024-12-10T11:03:50,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630983 2024-12-10T11:03:50,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630983) roll requested 2024-12-10T11:03:50,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630989 2024-12-10T11:03:50,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:50,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630989 2024-12-10T11:03:50,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:50,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:50,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630989) roll requested 2024-12-10T11:03:50,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828630996 2024-12-10T11:03:51,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630996 2024-12-10T11:03:51,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828630996) roll requested 2024-12-10T11:03:51,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631002 2024-12-10T11:03:51,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828630996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631002 2024-12-10T11:03:51,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631002) roll requested 2024-12-10T11:03:51,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631009 2024-12-10T11:03:51,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631009 2024-12-10T11:03:51,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631009) roll requested 2024-12-10T11:03:51,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631015 2024-12-10T11:03:51,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631015 2024-12-10T11:03:51,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631015) roll requested 2024-12-10T11:03:51,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631021 2024-12-10T11:03:51,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631021 2024-12-10T11:03:51,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631021) roll requested 2024-12-10T11:03:51,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631028 2024-12-10T11:03:51,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631028 2024-12-10T11:03:51,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631028) roll requested 2024-12-10T11:03:51,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631034 2024-12-10T11:03:51,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631034 2024-12-10T11:03:51,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631034) roll requested 2024-12-10T11:03:51,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631042 2024-12-10T11:03:51,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631042 2024-12-10T11:03:51,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631042) roll requested 2024-12-10T11:03:51,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631048 2024-12-10T11:03:51,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631048 2024-12-10T11:03:51,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631048) roll requested 2024-12-10T11:03:51,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631055 2024-12-10T11:03:51,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631055 2024-12-10T11:03:51,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631055) roll requested 2024-12-10T11:03:51,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631061 2024-12-10T11:03:51,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631061 2024-12-10T11:03:51,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631061) roll requested 2024-12-10T11:03:51,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631068 2024-12-10T11:03:51,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631068 2024-12-10T11:03:51,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631068) roll requested 2024-12-10T11:03:51,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631074 2024-12-10T11:03:51,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631074 2024-12-10T11:03:51,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631074) roll requested 2024-12-10T11:03:51,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631080 2024-12-10T11:03:51,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631080 2024-12-10T11:03:51,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631080) roll requested 2024-12-10T11:03:51,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631087 2024-12-10T11:03:51,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631087 2024-12-10T11:03:51,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631087) roll requested 2024-12-10T11:03:51,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631093 2024-12-10T11:03:51,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631093 2024-12-10T11:03:51,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631093) roll requested 2024-12-10T11:03:51,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631100 2024-12-10T11:03:51,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631100 2024-12-10T11:03:51,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631100) roll requested 2024-12-10T11:03:51,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631106 2024-12-10T11:03:51,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631106 2024-12-10T11:03:51,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631106) roll requested 2024-12-10T11:03:51,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631112 2024-12-10T11:03:51,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631112 2024-12-10T11:03:51,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631112) roll requested 2024-12-10T11:03:51,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631119 2024-12-10T11:03:51,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631119 2024-12-10T11:03:51,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631119) roll requested 2024-12-10T11:03:51,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631125 2024-12-10T11:03:51,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631125 2024-12-10T11:03:51,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631125) roll requested 2024-12-10T11:03:51,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631131 2024-12-10T11:03:51,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631131 2024-12-10T11:03:51,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631131) roll requested 2024-12-10T11:03:51,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631138 2024-12-10T11:03:51,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631138 2024-12-10T11:03:51,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631138) roll requested 2024-12-10T11:03:51,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631145 2024-12-10T11:03:51,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631145 2024-12-10T11:03:51,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631145) roll requested 2024-12-10T11:03:51,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631151 2024-12-10T11:03:51,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631151 2024-12-10T11:03:51,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631151) roll requested 2024-12-10T11:03:51,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631157 2024-12-10T11:03:51,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631157 2024-12-10T11:03:51,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631157) roll requested 2024-12-10T11:03:51,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631164 2024-12-10T11:03:51,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631164 2024-12-10T11:03:51,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631164) roll requested 2024-12-10T11:03:51,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631170 2024-12-10T11:03:51,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631170 2024-12-10T11:03:51,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631170) roll requested 2024-12-10T11:03:51,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631177 2024-12-10T11:03:51,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631177 2024-12-10T11:03:51,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631177) roll requested 2024-12-10T11:03:51,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631183 2024-12-10T11:03:51,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631183 2024-12-10T11:03:51,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631183) roll requested 2024-12-10T11:03:51,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631190 2024-12-10T11:03:51,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631190 2024-12-10T11:03:51,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631190) roll requested 2024-12-10T11:03:51,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631196 2024-12-10T11:03:51,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631196 2024-12-10T11:03:51,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631196) roll requested 2024-12-10T11:03:51,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631202 2024-12-10T11:03:51,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631202 2024-12-10T11:03:51,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631202) roll requested 2024-12-10T11:03:51,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631208 2024-12-10T11:03:51,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631208 2024-12-10T11:03:51,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631208) roll requested 2024-12-10T11:03:51,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631215 2024-12-10T11:03:51,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631215 2024-12-10T11:03:51,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631215) roll requested 2024-12-10T11:03:51,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631226 2024-12-10T11:03:51,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631226 2024-12-10T11:03:51,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631226) roll requested 2024-12-10T11:03:51,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631232 2024-12-10T11:03:51,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631232 2024-12-10T11:03:51,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631232) roll requested 2024-12-10T11:03:51,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631238 2024-12-10T11:03:51,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631238 2024-12-10T11:03:51,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631238) roll requested 2024-12-10T11:03:51,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631245 2024-12-10T11:03:51,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631245 2024-12-10T11:03:51,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631245) roll requested 2024-12-10T11:03:51,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631251 2024-12-10T11:03:51,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631251 2024-12-10T11:03:51,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631251) roll requested 2024-12-10T11:03:51,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631257 2024-12-10T11:03:51,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631257 2024-12-10T11:03:51,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631257) roll requested 2024-12-10T11:03:51,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631263 2024-12-10T11:03:51,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631263 2024-12-10T11:03:51,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631263) roll requested 2024-12-10T11:03:51,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631269 2024-12-10T11:03:51,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631269 2024-12-10T11:03:51,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631269) roll requested 2024-12-10T11:03:51,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631276 2024-12-10T11:03:51,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631276 2024-12-10T11:03:51,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631276) roll requested 2024-12-10T11:03:51,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631282 2024-12-10T11:03:51,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631282 2024-12-10T11:03:51,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631282) roll requested 2024-12-10T11:03:51,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631288 2024-12-10T11:03:51,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631288 2024-12-10T11:03:51,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631288) roll requested 2024-12-10T11:03:51,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631294 2024-12-10T11:03:51,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631294 2024-12-10T11:03:51,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631294) roll requested 2024-12-10T11:03:51,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631301 2024-12-10T11:03:51,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631301 2024-12-10T11:03:51,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631301) roll requested 2024-12-10T11:03:51,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631307 2024-12-10T11:03:51,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631307 2024-12-10T11:03:51,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631307) roll requested 2024-12-10T11:03:51,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631313 2024-12-10T11:03:51,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631313 2024-12-10T11:03:51,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631313) roll requested 2024-12-10T11:03:51,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631319 2024-12-10T11:03:51,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631319 2024-12-10T11:03:51,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631319) roll requested 2024-12-10T11:03:51,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631329 2024-12-10T11:03:51,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631329 2024-12-10T11:03:51,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631329) roll requested 2024-12-10T11:03:51,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631336 2024-12-10T11:03:51,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631336 2024-12-10T11:03:51,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631336) roll requested 2024-12-10T11:03:51,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631342 2024-12-10T11:03:51,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631342 2024-12-10T11:03:51,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631342) roll requested 2024-12-10T11:03:51,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631349 2024-12-10T11:03:51,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631349 2024-12-10T11:03:51,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631349) roll requested 2024-12-10T11:03:51,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631355 2024-12-10T11:03:51,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631355 2024-12-10T11:03:51,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631355) roll requested 2024-12-10T11:03:51,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631361 2024-12-10T11:03:51,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631361 2024-12-10T11:03:51,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631361) roll requested 2024-12-10T11:03:51,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631367 2024-12-10T11:03:51,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631367 2024-12-10T11:03:51,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631367) roll requested 2024-12-10T11:03:51,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631379 2024-12-10T11:03:51,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631379 2024-12-10T11:03:51,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631379) roll requested 2024-12-10T11:03:51,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631386 2024-12-10T11:03:51,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631386 2024-12-10T11:03:51,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631386) roll requested 2024-12-10T11:03:51,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631393 2024-12-10T11:03:51,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631393 2024-12-10T11:03:51,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631393) roll requested 2024-12-10T11:03:51,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631400 2024-12-10T11:03:51,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631400 2024-12-10T11:03:51,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631400) roll requested 2024-12-10T11:03:51,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631406 2024-12-10T11:03:51,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631406 2024-12-10T11:03:51,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631406) roll requested 2024-12-10T11:03:51,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631412 2024-12-10T11:03:51,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631412 2024-12-10T11:03:51,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631412) roll requested 2024-12-10T11:03:51,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631418 2024-12-10T11:03:51,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631418 2024-12-10T11:03:51,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631418) roll requested 2024-12-10T11:03:51,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631424 2024-12-10T11:03:51,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631424 2024-12-10T11:03:51,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631424) roll requested 2024-12-10T11:03:51,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631431 2024-12-10T11:03:51,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631431 2024-12-10T11:03:51,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631431) roll requested 2024-12-10T11:03:51,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631437 2024-12-10T11:03:51,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631437 2024-12-10T11:03:51,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631437) roll requested 2024-12-10T11:03:51,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631443 2024-12-10T11:03:51,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631443 2024-12-10T11:03:51,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631443) roll requested 2024-12-10T11:03:51,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631449 2024-12-10T11:03:51,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631449 2024-12-10T11:03:51,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631449) roll requested 2024-12-10T11:03:51,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631456 2024-12-10T11:03:51,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631456 2024-12-10T11:03:51,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631456) roll requested 2024-12-10T11:03:51,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631462 2024-12-10T11:03:51,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631462 2024-12-10T11:03:51,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631462) roll requested 2024-12-10T11:03:51,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631468 2024-12-10T11:03:51,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631468 2024-12-10T11:03:51,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631468) roll requested 2024-12-10T11:03:51,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631474 2024-12-10T11:03:51,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631474 2024-12-10T11:03:51,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631474) roll requested 2024-12-10T11:03:51,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631484 2024-12-10T11:03:51,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631484 2024-12-10T11:03:51,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631484) roll requested 2024-12-10T11:03:51,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631490 2024-12-10T11:03:51,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631490 2024-12-10T11:03:51,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631490) roll requested 2024-12-10T11:03:51,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631496 2024-12-10T11:03:51,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631496 2024-12-10T11:03:51,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631496) roll requested 2024-12-10T11:03:51,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631503 2024-12-10T11:03:51,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631503 2024-12-10T11:03:51,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631503) roll requested 2024-12-10T11:03:51,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631509 2024-12-10T11:03:51,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631509 2024-12-10T11:03:51,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631509) roll requested 2024-12-10T11:03:51,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631515 2024-12-10T11:03:51,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631515 2024-12-10T11:03:51,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631515) roll requested 2024-12-10T11:03:51,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631522 2024-12-10T11:03:51,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631522 2024-12-10T11:03:51,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631522) roll requested 2024-12-10T11:03:51,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631528 2024-12-10T11:03:51,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631528 2024-12-10T11:03:51,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631528) roll requested 2024-12-10T11:03:51,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631534 2024-12-10T11:03:51,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631534 2024-12-10T11:03:51,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631534) roll requested 2024-12-10T11:03:51,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631541 2024-12-10T11:03:51,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631541 2024-12-10T11:03:51,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631541) roll requested 2024-12-10T11:03:51,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631548 2024-12-10T11:03:51,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631548 2024-12-10T11:03:51,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631548) roll requested 2024-12-10T11:03:51,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631557 2024-12-10T11:03:51,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631557 2024-12-10T11:03:51,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631557) roll requested 2024-12-10T11:03:51,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631563 2024-12-10T11:03:51,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631563 2024-12-10T11:03:51,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631563) roll requested 2024-12-10T11:03:51,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631570 2024-12-10T11:03:51,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631570 2024-12-10T11:03:51,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631570) roll requested 2024-12-10T11:03:51,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631576 2024-12-10T11:03:51,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631576 2024-12-10T11:03:51,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631576) roll requested 2024-12-10T11:03:51,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631583 2024-12-10T11:03:51,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631583 2024-12-10T11:03:51,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631583) roll requested 2024-12-10T11:03:51,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631589 2024-12-10T11:03:51,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631589 2024-12-10T11:03:51,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631589) roll requested 2024-12-10T11:03:51,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631599 2024-12-10T11:03:51,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631599 2024-12-10T11:03:51,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631599) roll requested 2024-12-10T11:03:51,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631607 2024-12-10T11:03:51,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631607 2024-12-10T11:03:51,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631607) roll requested 2024-12-10T11:03:51,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631614 2024-12-10T11:03:51,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631614 2024-12-10T11:03:51,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631614) roll requested 2024-12-10T11:03:51,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631620 2024-12-10T11:03:51,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631620 2024-12-10T11:03:51,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631620) roll requested 2024-12-10T11:03:51,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631626 2024-12-10T11:03:51,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631626 2024-12-10T11:03:51,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631626) roll requested 2024-12-10T11:03:51,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631632 2024-12-10T11:03:51,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631632 2024-12-10T11:03:51,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631632) roll requested 2024-12-10T11:03:51,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631639 2024-12-10T11:03:51,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631639 2024-12-10T11:03:51,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631639) roll requested 2024-12-10T11:03:51,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631646 2024-12-10T11:03:51,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631646 2024-12-10T11:03:51,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631646) roll requested 2024-12-10T11:03:51,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631653 2024-12-10T11:03:51,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631653 2024-12-10T11:03:51,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631653) roll requested 2024-12-10T11:03:51,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631660 2024-12-10T11:03:51,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631660 2024-12-10T11:03:51,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631660) roll requested 2024-12-10T11:03:51,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631669 2024-12-10T11:03:51,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631669 2024-12-10T11:03:51,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631669) roll requested 2024-12-10T11:03:51,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631676 2024-12-10T11:03:51,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631676 2024-12-10T11:03:51,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631676) roll requested 2024-12-10T11:03:51,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631685 2024-12-10T11:03:51,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631685 2024-12-10T11:03:51,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631685) roll requested 2024-12-10T11:03:51,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631694 2024-12-10T11:03:51,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631694 2024-12-10T11:03:51,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631694) roll requested 2024-12-10T11:03:51,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631701 2024-12-10T11:03:51,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631701 2024-12-10T11:03:51,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631701) roll requested 2024-12-10T11:03:51,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631710 2024-12-10T11:03:51,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631710 2024-12-10T11:03:51,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631710) roll requested 2024-12-10T11:03:51,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631717 2024-12-10T11:03:51,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631717 2024-12-10T11:03:51,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631717) roll requested 2024-12-10T11:03:51,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631723 2024-12-10T11:03:51,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631723 2024-12-10T11:03:51,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631723) roll requested 2024-12-10T11:03:51,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631730 2024-12-10T11:03:51,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631730 2024-12-10T11:03:51,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631730) roll requested 2024-12-10T11:03:51,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631738 2024-12-10T11:03:51,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631738 2024-12-10T11:03:51,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631738) roll requested 2024-12-10T11:03:51,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631745 2024-12-10T11:03:51,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631745 2024-12-10T11:03:51,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631745) roll requested 2024-12-10T11:03:51,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631753 2024-12-10T11:03:51,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631753 2024-12-10T11:03:51,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631753) roll requested 2024-12-10T11:03:51,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631760 2024-12-10T11:03:51,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631760 2024-12-10T11:03:51,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631760) roll requested 2024-12-10T11:03:51,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631767 2024-12-10T11:03:51,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631767 2024-12-10T11:03:51,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631767) roll requested 2024-12-10T11:03:51,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631773 2024-12-10T11:03:51,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631773 2024-12-10T11:03:51,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631773) roll requested 2024-12-10T11:03:51,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631780 2024-12-10T11:03:51,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631780 2024-12-10T11:03:51,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631780) roll requested 2024-12-10T11:03:51,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631786 2024-12-10T11:03:51,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631786 2024-12-10T11:03:51,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631786) roll requested 2024-12-10T11:03:51,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631793 2024-12-10T11:03:51,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631793 2024-12-10T11:03:51,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631793) roll requested 2024-12-10T11:03:51,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631799 2024-12-10T11:03:51,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631799 2024-12-10T11:03:51,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631799) roll requested 2024-12-10T11:03:51,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631805 2024-12-10T11:03:51,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631805 2024-12-10T11:03:51,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631805) roll requested 2024-12-10T11:03:51,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631812 2024-12-10T11:03:51,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631812 2024-12-10T11:03:51,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631812) roll requested 2024-12-10T11:03:51,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631819 2024-12-10T11:03:51,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631819 2024-12-10T11:03:51,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631819) roll requested 2024-12-10T11:03:51,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631825 2024-12-10T11:03:51,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631825 2024-12-10T11:03:51,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631825) roll requested 2024-12-10T11:03:51,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631831 2024-12-10T11:03:51,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631831 2024-12-10T11:03:51,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631831) roll requested 2024-12-10T11:03:51,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631838 2024-12-10T11:03:51,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631838 2024-12-10T11:03:51,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631838) roll requested 2024-12-10T11:03:51,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631846 2024-12-10T11:03:51,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631846 2024-12-10T11:03:51,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631846) roll requested 2024-12-10T11:03:51,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631857 2024-12-10T11:03:51,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631857 2024-12-10T11:03:51,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631857) roll requested 2024-12-10T11:03:51,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631864 2024-12-10T11:03:51,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631864 2024-12-10T11:03:51,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631864) roll requested 2024-12-10T11:03:51,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631870 2024-12-10T11:03:51,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631870 2024-12-10T11:03:51,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631870) roll requested 2024-12-10T11:03:51,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631876 2024-12-10T11:03:51,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631876 2024-12-10T11:03:51,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631876) roll requested 2024-12-10T11:03:51,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631883 2024-12-10T11:03:51,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631883 2024-12-10T11:03:51,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631883) roll requested 2024-12-10T11:03:51,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631889 2024-12-10T11:03:51,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631889 2024-12-10T11:03:51,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631889) roll requested 2024-12-10T11:03:51,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631896 2024-12-10T11:03:51,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631896 2024-12-10T11:03:51,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631896) roll requested 2024-12-10T11:03:51,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631902 2024-12-10T11:03:51,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631902 2024-12-10T11:03:51,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631902) roll requested 2024-12-10T11:03:51,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631908 2024-12-10T11:03:51,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631908 2024-12-10T11:03:51,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631908) roll requested 2024-12-10T11:03:51,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631914 2024-12-10T11:03:51,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631914 2024-12-10T11:03:51,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631914) roll requested 2024-12-10T11:03:51,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631920 2024-12-10T11:03:51,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631920 2024-12-10T11:03:51,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631920) roll requested 2024-12-10T11:03:51,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631927 2024-12-10T11:03:51,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631927 2024-12-10T11:03:51,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631927) roll requested 2024-12-10T11:03:51,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631933 2024-12-10T11:03:51,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631933 2024-12-10T11:03:51,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631933) roll requested 2024-12-10T11:03:51,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631939 2024-12-10T11:03:51,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631939 2024-12-10T11:03:51,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631939) roll requested 2024-12-10T11:03:51,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631945 2024-12-10T11:03:51,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631945 2024-12-10T11:03:51,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631945) roll requested 2024-12-10T11:03:51,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631951 2024-12-10T11:03:51,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631951 2024-12-10T11:03:51,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631951) roll requested 2024-12-10T11:03:51,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631963 2024-12-10T11:03:51,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631963 2024-12-10T11:03:51,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631963) roll requested 2024-12-10T11:03:51,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631969 2024-12-10T11:03:51,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631969 2024-12-10T11:03:51,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631969) roll requested 2024-12-10T11:03:51,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631975 2024-12-10T11:03:51,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631975 2024-12-10T11:03:51,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631975) roll requested 2024-12-10T11:03:51,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631981 2024-12-10T11:03:51,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631981 2024-12-10T11:03:51,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631981) roll requested 2024-12-10T11:03:51,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631988 2024-12-10T11:03:51,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631988 2024-12-10T11:03:51,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:51,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:51,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631988) roll requested 2024-12-10T11:03:51,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828631994 2024-12-10T11:03:51,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:51,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631994 2024-12-10T11:03:52,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828631994) roll requested 2024-12-10T11:03:52,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632000 2024-12-10T11:03:52,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828631994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632000 2024-12-10T11:03:52,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632000) roll requested 2024-12-10T11:03:52,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632006 2024-12-10T11:03:52,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632006 2024-12-10T11:03:52,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632006) roll requested 2024-12-10T11:03:52,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632012 2024-12-10T11:03:52,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632012 2024-12-10T11:03:52,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632012) roll requested 2024-12-10T11:03:52,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632018 2024-12-10T11:03:52,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632018 2024-12-10T11:03:52,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632018) roll requested 2024-12-10T11:03:52,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632025 2024-12-10T11:03:52,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632025 2024-12-10T11:03:52,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632025) roll requested 2024-12-10T11:03:52,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632031 2024-12-10T11:03:52,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632031 2024-12-10T11:03:52,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632031) roll requested 2024-12-10T11:03:52,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632037 2024-12-10T11:03:52,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632037 2024-12-10T11:03:52,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632037) roll requested 2024-12-10T11:03:52,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632043 2024-12-10T11:03:52,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632043 2024-12-10T11:03:52,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632043) roll requested 2024-12-10T11:03:52,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632049 2024-12-10T11:03:52,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632049 2024-12-10T11:03:52,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632049) roll requested 2024-12-10T11:03:52,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632055 2024-12-10T11:03:52,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632055 2024-12-10T11:03:52,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632055) roll requested 2024-12-10T11:03:52,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632061 2024-12-10T11:03:52,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632061 2024-12-10T11:03:52,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632061) roll requested 2024-12-10T11:03:52,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632072 2024-12-10T11:03:52,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632072 2024-12-10T11:03:52,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632072) roll requested 2024-12-10T11:03:52,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632079 2024-12-10T11:03:52,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632079 2024-12-10T11:03:52,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632079) roll requested 2024-12-10T11:03:52,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632085 2024-12-10T11:03:52,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632085 2024-12-10T11:03:52,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632085) roll requested 2024-12-10T11:03:52,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632091 2024-12-10T11:03:52,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632091 2024-12-10T11:03:52,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632091) roll requested 2024-12-10T11:03:52,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632098 2024-12-10T11:03:52,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632098 2024-12-10T11:03:52,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632098) roll requested 2024-12-10T11:03:52,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632104 2024-12-10T11:03:52,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632104 2024-12-10T11:03:52,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632104) roll requested 2024-12-10T11:03:52,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632110 2024-12-10T11:03:52,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632110 2024-12-10T11:03:52,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632110) roll requested 2024-12-10T11:03:52,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632116 2024-12-10T11:03:52,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632116 2024-12-10T11:03:52,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632116) roll requested 2024-12-10T11:03:52,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632122 2024-12-10T11:03:52,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632122 2024-12-10T11:03:52,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632122) roll requested 2024-12-10T11:03:52,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632129 2024-12-10T11:03:52,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632129 2024-12-10T11:03:52,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632129) roll requested 2024-12-10T11:03:52,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632135 2024-12-10T11:03:52,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632135 2024-12-10T11:03:52,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632135) roll requested 2024-12-10T11:03:52,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632141 2024-12-10T11:03:52,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632141 2024-12-10T11:03:52,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632141) roll requested 2024-12-10T11:03:52,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632148 2024-12-10T11:03:52,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632148 2024-12-10T11:03:52,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632148) roll requested 2024-12-10T11:03:52,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632154 2024-12-10T11:03:52,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632154 2024-12-10T11:03:52,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632154) roll requested 2024-12-10T11:03:52,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632160 2024-12-10T11:03:52,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632160 2024-12-10T11:03:52,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632160) roll requested 2024-12-10T11:03:52,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632166 2024-12-10T11:03:52,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632166 2024-12-10T11:03:52,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632166) roll requested 2024-12-10T11:03:52,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632173 2024-12-10T11:03:52,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632173 2024-12-10T11:03:52,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632173) roll requested 2024-12-10T11:03:52,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632179 2024-12-10T11:03:52,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632179 2024-12-10T11:03:52,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632179) roll requested 2024-12-10T11:03:52,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632186 2024-12-10T11:03:52,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632186 2024-12-10T11:03:52,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632186) roll requested 2024-12-10T11:03:52,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632192 2024-12-10T11:03:52,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632192 2024-12-10T11:03:52,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632192) roll requested 2024-12-10T11:03:52,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632198 2024-12-10T11:03:52,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632198 2024-12-10T11:03:52,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632198) roll requested 2024-12-10T11:03:52,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632204 2024-12-10T11:03:52,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632204 2024-12-10T11:03:52,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632204) roll requested 2024-12-10T11:03:52,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632211 2024-12-10T11:03:52,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632211 2024-12-10T11:03:52,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632211) roll requested 2024-12-10T11:03:52,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632217 2024-12-10T11:03:52,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632217 2024-12-10T11:03:52,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632217) roll requested 2024-12-10T11:03:52,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632223 2024-12-10T11:03:52,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632223 2024-12-10T11:03:52,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632223) roll requested 2024-12-10T11:03:52,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632229 2024-12-10T11:03:52,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632229 2024-12-10T11:03:52,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632229) roll requested 2024-12-10T11:03:52,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632236 2024-12-10T11:03:52,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632236 2024-12-10T11:03:52,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632236) roll requested 2024-12-10T11:03:52,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632242 2024-12-10T11:03:52,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632242 2024-12-10T11:03:52,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632242) roll requested 2024-12-10T11:03:52,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632249 2024-12-10T11:03:52,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632249 2024-12-10T11:03:52,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632249) roll requested 2024-12-10T11:03:52,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632255 2024-12-10T11:03:52,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632255 2024-12-10T11:03:52,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632255) roll requested 2024-12-10T11:03:52,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632261 2024-12-10T11:03:52,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632261 2024-12-10T11:03:52,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632261) roll requested 2024-12-10T11:03:52,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632268 2024-12-10T11:03:52,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632268 2024-12-10T11:03:52,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632268) roll requested 2024-12-10T11:03:52,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632274 2024-12-10T11:03:52,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632274 2024-12-10T11:03:52,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632274) roll requested 2024-12-10T11:03:52,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632280 2024-12-10T11:03:52,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632280 2024-12-10T11:03:52,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632280) roll requested 2024-12-10T11:03:52,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632286 2024-12-10T11:03:52,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632286 2024-12-10T11:03:52,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632286) roll requested 2024-12-10T11:03:52,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632293 2024-12-10T11:03:52,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632293 2024-12-10T11:03:52,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632293) roll requested 2024-12-10T11:03:52,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632299 2024-12-10T11:03:52,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632299 2024-12-10T11:03:52,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632299) roll requested 2024-12-10T11:03:52,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632306 2024-12-10T11:03:52,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632306 2024-12-10T11:03:52,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632306) roll requested 2024-12-10T11:03:52,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632312 2024-12-10T11:03:52,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632312 2024-12-10T11:03:52,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632312) roll requested 2024-12-10T11:03:52,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632318 2024-12-10T11:03:52,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632318 2024-12-10T11:03:52,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632318) roll requested 2024-12-10T11:03:52,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632324 2024-12-10T11:03:52,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632324 2024-12-10T11:03:52,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632324) roll requested 2024-12-10T11:03:52,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632331 2024-12-10T11:03:52,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632331 2024-12-10T11:03:52,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632331) roll requested 2024-12-10T11:03:52,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632337 2024-12-10T11:03:52,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632337 2024-12-10T11:03:52,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632337) roll requested 2024-12-10T11:03:52,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632343 2024-12-10T11:03:52,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632343 2024-12-10T11:03:52,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632343) roll requested 2024-12-10T11:03:52,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632350 2024-12-10T11:03:52,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632350 2024-12-10T11:03:52,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632350) roll requested 2024-12-10T11:03:52,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632356 2024-12-10T11:03:52,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632356 2024-12-10T11:03:52,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632356) roll requested 2024-12-10T11:03:52,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632362 2024-12-10T11:03:52,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632362 2024-12-10T11:03:52,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632362) roll requested 2024-12-10T11:03:52,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632368 2024-12-10T11:03:52,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632368 2024-12-10T11:03:52,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632368) roll requested 2024-12-10T11:03:52,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632374 2024-12-10T11:03:52,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632374 2024-12-10T11:03:52,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632374) roll requested 2024-12-10T11:03:52,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632381 2024-12-10T11:03:52,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632381 2024-12-10T11:03:52,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632381) roll requested 2024-12-10T11:03:52,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632387 2024-12-10T11:03:52,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632387 2024-12-10T11:03:52,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632387) roll requested 2024-12-10T11:03:52,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632393 2024-12-10T11:03:52,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632393 2024-12-10T11:03:52,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632393) roll requested 2024-12-10T11:03:52,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632400 2024-12-10T11:03:52,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632400 2024-12-10T11:03:52,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632400) roll requested 2024-12-10T11:03:52,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632406 2024-12-10T11:03:52,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632406 2024-12-10T11:03:52,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632406) roll requested 2024-12-10T11:03:52,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632413 2024-12-10T11:03:52,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632413 2024-12-10T11:03:52,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632413) roll requested 2024-12-10T11:03:52,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632419 2024-12-10T11:03:52,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632419 2024-12-10T11:03:52,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632419) roll requested 2024-12-10T11:03:52,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632425 2024-12-10T11:03:52,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632425 2024-12-10T11:03:52,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632425) roll requested 2024-12-10T11:03:52,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632431 2024-12-10T11:03:52,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632431 2024-12-10T11:03:52,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632431) roll requested 2024-12-10T11:03:52,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632437 2024-12-10T11:03:52,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632437 2024-12-10T11:03:52,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632437) roll requested 2024-12-10T11:03:52,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632444 2024-12-10T11:03:52,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632444 2024-12-10T11:03:52,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632444) roll requested 2024-12-10T11:03:52,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632450 2024-12-10T11:03:52,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632450 2024-12-10T11:03:52,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632450) roll requested 2024-12-10T11:03:52,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632457 2024-12-10T11:03:52,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632457 2024-12-10T11:03:52,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632457) roll requested 2024-12-10T11:03:52,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632463 2024-12-10T11:03:52,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632463 2024-12-10T11:03:52,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632463) roll requested 2024-12-10T11:03:52,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632469 2024-12-10T11:03:52,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632469 2024-12-10T11:03:52,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632469) roll requested 2024-12-10T11:03:52,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632475 2024-12-10T11:03:52,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632475 2024-12-10T11:03:52,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632475) roll requested 2024-12-10T11:03:52,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632487 2024-12-10T11:03:52,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632487 2024-12-10T11:03:52,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632487) roll requested 2024-12-10T11:03:52,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632494 2024-12-10T11:03:52,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632494 2024-12-10T11:03:52,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632494) roll requested 2024-12-10T11:03:52,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632501 2024-12-10T11:03:52,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632501 2024-12-10T11:03:52,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632501) roll requested 2024-12-10T11:03:52,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632507 2024-12-10T11:03:52,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632507 2024-12-10T11:03:52,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632507) roll requested 2024-12-10T11:03:52,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632513 2024-12-10T11:03:52,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632513 2024-12-10T11:03:52,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632513) roll requested 2024-12-10T11:03:52,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632524 2024-12-10T11:03:52,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632524 2024-12-10T11:03:52,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632524) roll requested 2024-12-10T11:03:52,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632531 2024-12-10T11:03:52,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632531 2024-12-10T11:03:52,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632531) roll requested 2024-12-10T11:03:52,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632537 2024-12-10T11:03:52,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632537 2024-12-10T11:03:52,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632537) roll requested 2024-12-10T11:03:52,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632543 2024-12-10T11:03:52,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632543 2024-12-10T11:03:52,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632543) roll requested 2024-12-10T11:03:52,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632549 2024-12-10T11:03:52,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632549 2024-12-10T11:03:52,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632549) roll requested 2024-12-10T11:03:52,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632557 2024-12-10T11:03:52,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632557 2024-12-10T11:03:52,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632557) roll requested 2024-12-10T11:03:52,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632563 2024-12-10T11:03:52,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632563 2024-12-10T11:03:52,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632563) roll requested 2024-12-10T11:03:52,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632569 2024-12-10T11:03:52,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632569 2024-12-10T11:03:52,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632569) roll requested 2024-12-10T11:03:52,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632576 2024-12-10T11:03:52,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632576 2024-12-10T11:03:52,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632576) roll requested 2024-12-10T11:03:52,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632582 2024-12-10T11:03:52,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632582 2024-12-10T11:03:52,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632582) roll requested 2024-12-10T11:03:52,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632588 2024-12-10T11:03:52,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632588 2024-12-10T11:03:52,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632588) roll requested 2024-12-10T11:03:52,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632595 2024-12-10T11:03:52,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632595 2024-12-10T11:03:52,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632595) roll requested 2024-12-10T11:03:52,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632601 2024-12-10T11:03:52,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632601 2024-12-10T11:03:52,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632601) roll requested 2024-12-10T11:03:52,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632607 2024-12-10T11:03:52,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632607 2024-12-10T11:03:52,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632607) roll requested 2024-12-10T11:03:52,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632614 2024-12-10T11:03:52,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632614 2024-12-10T11:03:52,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632614) roll requested 2024-12-10T11:03:52,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632620 2024-12-10T11:03:52,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632620 2024-12-10T11:03:52,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632620) roll requested 2024-12-10T11:03:52,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632626 2024-12-10T11:03:52,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632626 2024-12-10T11:03:52,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632626) roll requested 2024-12-10T11:03:52,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632633 2024-12-10T11:03:52,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632633 2024-12-10T11:03:52,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632633) roll requested 2024-12-10T11:03:52,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632639 2024-12-10T11:03:52,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632639 2024-12-10T11:03:52,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632639) roll requested 2024-12-10T11:03:52,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632645 2024-12-10T11:03:52,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632645 2024-12-10T11:03:52,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632645) roll requested 2024-12-10T11:03:52,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632651 2024-12-10T11:03:52,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632651 2024-12-10T11:03:52,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632651) roll requested 2024-12-10T11:03:52,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632658 2024-12-10T11:03:52,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632658 2024-12-10T11:03:52,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632658) roll requested 2024-12-10T11:03:52,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632664 2024-12-10T11:03:52,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632664 2024-12-10T11:03:52,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632664) roll requested 2024-12-10T11:03:52,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632670 2024-12-10T11:03:52,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632670 2024-12-10T11:03:52,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632670) roll requested 2024-12-10T11:03:52,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632676 2024-12-10T11:03:52,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632676 2024-12-10T11:03:52,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632676) roll requested 2024-12-10T11:03:52,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632682 2024-12-10T11:03:52,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632682 2024-12-10T11:03:52,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632682) roll requested 2024-12-10T11:03:52,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632688 2024-12-10T11:03:52,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632688 2024-12-10T11:03:52,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632688) roll requested 2024-12-10T11:03:52,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632695 2024-12-10T11:03:52,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632695 2024-12-10T11:03:52,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632695) roll requested 2024-12-10T11:03:52,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632701 2024-12-10T11:03:52,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632701 2024-12-10T11:03:52,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632701) roll requested 2024-12-10T11:03:52,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632707 2024-12-10T11:03:52,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632707 2024-12-10T11:03:52,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632707) roll requested 2024-12-10T11:03:52,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632714 2024-12-10T11:03:52,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632714 2024-12-10T11:03:52,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632714) roll requested 2024-12-10T11:03:52,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632720 2024-12-10T11:03:52,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632720 2024-12-10T11:03:52,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632720) roll requested 2024-12-10T11:03:52,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632727 2024-12-10T11:03:52,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632727 2024-12-10T11:03:52,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632727) roll requested 2024-12-10T11:03:52,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632733 2024-12-10T11:03:52,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632733 2024-12-10T11:03:52,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632733) roll requested 2024-12-10T11:03:52,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632739 2024-12-10T11:03:52,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632739 2024-12-10T11:03:52,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632739) roll requested 2024-12-10T11:03:52,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632745 2024-12-10T11:03:52,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632745 2024-12-10T11:03:52,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632745) roll requested 2024-12-10T11:03:52,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632751 2024-12-10T11:03:52,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632751 2024-12-10T11:03:52,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632751) roll requested 2024-12-10T11:03:52,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632757 2024-12-10T11:03:52,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632757 2024-12-10T11:03:52,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632757) roll requested 2024-12-10T11:03:52,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632763 2024-12-10T11:03:52,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632763 2024-12-10T11:03:52,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632763) roll requested 2024-12-10T11:03:52,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632769 2024-12-10T11:03:52,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632769 2024-12-10T11:03:52,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632769) roll requested 2024-12-10T11:03:52,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632775 2024-12-10T11:03:52,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632775 2024-12-10T11:03:52,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632775) roll requested 2024-12-10T11:03:52,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632782 2024-12-10T11:03:52,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632782 2024-12-10T11:03:52,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632782) roll requested 2024-12-10T11:03:52,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632788 2024-12-10T11:03:52,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632788 2024-12-10T11:03:52,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632788) roll requested 2024-12-10T11:03:52,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632794 2024-12-10T11:03:52,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632794 2024-12-10T11:03:52,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632794) roll requested 2024-12-10T11:03:52,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632800 2024-12-10T11:03:52,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632800 2024-12-10T11:03:52,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632800) roll requested 2024-12-10T11:03:52,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632806 2024-12-10T11:03:52,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632806 2024-12-10T11:03:52,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632806) roll requested 2024-12-10T11:03:52,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632813 2024-12-10T11:03:52,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632813 2024-12-10T11:03:52,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632813) roll requested 2024-12-10T11:03:52,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632819 2024-12-10T11:03:52,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632819 2024-12-10T11:03:52,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632819) roll requested 2024-12-10T11:03:52,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632829 2024-12-10T11:03:52,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632829 2024-12-10T11:03:52,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632829) roll requested 2024-12-10T11:03:52,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632836 2024-12-10T11:03:52,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632836 2024-12-10T11:03:52,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632836) roll requested 2024-12-10T11:03:52,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632842 2024-12-10T11:03:52,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632842 2024-12-10T11:03:52,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632842) roll requested 2024-12-10T11:03:52,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632848 2024-12-10T11:03:52,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632848 2024-12-10T11:03:52,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632848) roll requested 2024-12-10T11:03:52,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632863 2024-12-10T11:03:52,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632863 2024-12-10T11:03:52,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632863) roll requested 2024-12-10T11:03:52,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632870 2024-12-10T11:03:52,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632870 2024-12-10T11:03:52,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632870) roll requested 2024-12-10T11:03:52,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632880 2024-12-10T11:03:52,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632880 2024-12-10T11:03:52,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632880) roll requested 2024-12-10T11:03:52,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632889 2024-12-10T11:03:52,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632889 2024-12-10T11:03:52,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632889) roll requested 2024-12-10T11:03:52,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632895 2024-12-10T11:03:52,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632895 2024-12-10T11:03:52,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632895) roll requested 2024-12-10T11:03:52,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632902 2024-12-10T11:03:52,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632902 2024-12-10T11:03:52,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632902) roll requested 2024-12-10T11:03:52,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632908 2024-12-10T11:03:52,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632908 2024-12-10T11:03:52,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632908) roll requested 2024-12-10T11:03:52,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632914 2024-12-10T11:03:52,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632914 2024-12-10T11:03:52,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632914) roll requested 2024-12-10T11:03:52,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632921 2024-12-10T11:03:52,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632921 2024-12-10T11:03:52,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632921) roll requested 2024-12-10T11:03:52,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632927 2024-12-10T11:03:52,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632927 2024-12-10T11:03:52,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632927) roll requested 2024-12-10T11:03:52,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632933 2024-12-10T11:03:52,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632933 2024-12-10T11:03:52,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632933) roll requested 2024-12-10T11:03:52,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632940 2024-12-10T11:03:52,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632940 2024-12-10T11:03:52,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632940) roll requested 2024-12-10T11:03:52,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632946 2024-12-10T11:03:52,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632946 2024-12-10T11:03:52,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632946) roll requested 2024-12-10T11:03:52,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632952 2024-12-10T11:03:52,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632952 2024-12-10T11:03:52,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632952) roll requested 2024-12-10T11:03:52,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632959 2024-12-10T11:03:52,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632959 2024-12-10T11:03:52,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632959) roll requested 2024-12-10T11:03:52,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632965 2024-12-10T11:03:52,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632965 2024-12-10T11:03:52,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632965) roll requested 2024-12-10T11:03:52,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632971 2024-12-10T11:03:52,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632971 2024-12-10T11:03:52,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632971) roll requested 2024-12-10T11:03:52,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632977 2024-12-10T11:03:52,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632977 2024-12-10T11:03:52,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632977) roll requested 2024-12-10T11:03:52,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632983 2024-12-10T11:03:52,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:52,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632983 2024-12-10T11:03:52,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:52,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:52,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632983) roll requested 2024-12-10T11:03:52,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828632998 2024-12-10T11:03:53,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632998 2024-12-10T11:03:53,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828632998) roll requested 2024-12-10T11:03:53,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633006 2024-12-10T11:03:53,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828632998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633006 2024-12-10T11:03:53,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633006) roll requested 2024-12-10T11:03:53,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633013 2024-12-10T11:03:53,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633013 2024-12-10T11:03:53,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633013) roll requested 2024-12-10T11:03:53,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633023 2024-12-10T11:03:53,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633023 2024-12-10T11:03:53,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633023) roll requested 2024-12-10T11:03:53,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633031 2024-12-10T11:03:53,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633031 2024-12-10T11:03:53,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633031) roll requested 2024-12-10T11:03:53,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633037 2024-12-10T11:03:53,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633037 2024-12-10T11:03:53,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633037) roll requested 2024-12-10T11:03:53,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633044 2024-12-10T11:03:53,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633044 2024-12-10T11:03:53,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633044) roll requested 2024-12-10T11:03:53,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633050 2024-12-10T11:03:53,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633050 2024-12-10T11:03:53,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633050) roll requested 2024-12-10T11:03:53,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633056 2024-12-10T11:03:53,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633056 2024-12-10T11:03:53,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633056) roll requested 2024-12-10T11:03:53,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633062 2024-12-10T11:03:53,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633062 2024-12-10T11:03:53,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633062) roll requested 2024-12-10T11:03:53,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633069 2024-12-10T11:03:53,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633069 2024-12-10T11:03:53,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633069) roll requested 2024-12-10T11:03:53,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633075 2024-12-10T11:03:53,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633075 2024-12-10T11:03:53,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633075) roll requested 2024-12-10T11:03:53,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633081 2024-12-10T11:03:53,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633081 2024-12-10T11:03:53,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633081) roll requested 2024-12-10T11:03:53,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633087 2024-12-10T11:03:53,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633087 2024-12-10T11:03:53,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633087) roll requested 2024-12-10T11:03:53,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633094 2024-12-10T11:03:53,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633094 2024-12-10T11:03:53,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633094) roll requested 2024-12-10T11:03:53,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633100 2024-12-10T11:03:53,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633100 2024-12-10T11:03:53,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633100) roll requested 2024-12-10T11:03:53,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633106 2024-12-10T11:03:53,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633106 2024-12-10T11:03:53,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633106) roll requested 2024-12-10T11:03:53,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633116 2024-12-10T11:03:53,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633116 2024-12-10T11:03:53,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633116) roll requested 2024-12-10T11:03:53,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633123 2024-12-10T11:03:53,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633123 2024-12-10T11:03:53,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633123) roll requested 2024-12-10T11:03:53,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633129 2024-12-10T11:03:53,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633129 2024-12-10T11:03:53,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633129) roll requested 2024-12-10T11:03:53,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633136 2024-12-10T11:03:53,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633136 2024-12-10T11:03:53,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633136) roll requested 2024-12-10T11:03:53,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633142 2024-12-10T11:03:53,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633142 2024-12-10T11:03:53,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633142) roll requested 2024-12-10T11:03:53,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633148 2024-12-10T11:03:53,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633148 2024-12-10T11:03:53,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633148) roll requested 2024-12-10T11:03:53,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633154 2024-12-10T11:03:53,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633154 2024-12-10T11:03:53,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633154) roll requested 2024-12-10T11:03:53,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633160 2024-12-10T11:03:53,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633160 2024-12-10T11:03:53,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633160) roll requested 2024-12-10T11:03:53,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633167 2024-12-10T11:03:53,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633167 2024-12-10T11:03:53,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633167) roll requested 2024-12-10T11:03:53,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633173 2024-12-10T11:03:53,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633173 2024-12-10T11:03:53,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633173) roll requested 2024-12-10T11:03:53,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633179 2024-12-10T11:03:53,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633179 2024-12-10T11:03:53,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633179) roll requested 2024-12-10T11:03:53,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633186 2024-12-10T11:03:53,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633186 2024-12-10T11:03:53,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633186) roll requested 2024-12-10T11:03:53,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633192 2024-12-10T11:03:53,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633192 2024-12-10T11:03:53,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633192) roll requested 2024-12-10T11:03:53,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633198 2024-12-10T11:03:53,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633198 2024-12-10T11:03:53,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633198) roll requested 2024-12-10T11:03:53,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633204 2024-12-10T11:03:53,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633204 2024-12-10T11:03:53,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633204) roll requested 2024-12-10T11:03:53,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633211 2024-12-10T11:03:53,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633211 2024-12-10T11:03:53,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633211) roll requested 2024-12-10T11:03:53,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633217 2024-12-10T11:03:53,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633217 2024-12-10T11:03:53,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633217) roll requested 2024-12-10T11:03:53,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633223 2024-12-10T11:03:53,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633223 2024-12-10T11:03:53,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633223) roll requested 2024-12-10T11:03:53,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633230 2024-12-10T11:03:53,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633230 2024-12-10T11:03:53,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633230) roll requested 2024-12-10T11:03:53,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633236 2024-12-10T11:03:53,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633236 2024-12-10T11:03:53,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633236) roll requested 2024-12-10T11:03:53,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633242 2024-12-10T11:03:53,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633242 2024-12-10T11:03:53,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633242) roll requested 2024-12-10T11:03:53,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633248 2024-12-10T11:03:53,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633248 2024-12-10T11:03:53,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633248) roll requested 2024-12-10T11:03:53,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633254 2024-12-10T11:03:53,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633254 2024-12-10T11:03:53,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633254) roll requested 2024-12-10T11:03:53,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633261 2024-12-10T11:03:53,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633261 2024-12-10T11:03:53,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633261) roll requested 2024-12-10T11:03:53,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633267 2024-12-10T11:03:53,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633267 2024-12-10T11:03:53,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633267) roll requested 2024-12-10T11:03:53,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633273 2024-12-10T11:03:53,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633273 2024-12-10T11:03:53,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633273) roll requested 2024-12-10T11:03:53,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633279 2024-12-10T11:03:53,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633279 2024-12-10T11:03:53,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633279) roll requested 2024-12-10T11:03:53,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633285 2024-12-10T11:03:53,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633285 2024-12-10T11:03:53,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633285) roll requested 2024-12-10T11:03:53,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633293 2024-12-10T11:03:53,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633293 2024-12-10T11:03:53,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633293) roll requested 2024-12-10T11:03:53,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633301 2024-12-10T11:03:53,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633301 2024-12-10T11:03:53,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633301) roll requested 2024-12-10T11:03:53,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633307 2024-12-10T11:03:53,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633307 2024-12-10T11:03:53,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633307) roll requested 2024-12-10T11:03:53,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633314 2024-12-10T11:03:53,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633314 2024-12-10T11:03:53,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633314) roll requested 2024-12-10T11:03:53,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633322 2024-12-10T11:03:53,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633322 2024-12-10T11:03:53,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633322) roll requested 2024-12-10T11:03:53,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633328 2024-12-10T11:03:53,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633328 2024-12-10T11:03:53,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633328) roll requested 2024-12-10T11:03:53,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633334 2024-12-10T11:03:53,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633334 2024-12-10T11:03:53,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633334) roll requested 2024-12-10T11:03:53,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633342 2024-12-10T11:03:53,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633342 2024-12-10T11:03:53,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633342) roll requested 2024-12-10T11:03:53,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633348 2024-12-10T11:03:53,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633348 2024-12-10T11:03:53,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633348) roll requested 2024-12-10T11:03:53,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633355 2024-12-10T11:03:53,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633355 2024-12-10T11:03:53,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633355) roll requested 2024-12-10T11:03:53,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633364 2024-12-10T11:03:53,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633364 2024-12-10T11:03:53,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633364) roll requested 2024-12-10T11:03:53,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633372 2024-12-10T11:03:53,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633372 2024-12-10T11:03:53,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633372) roll requested 2024-12-10T11:03:53,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633380 2024-12-10T11:03:53,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633380 2024-12-10T11:03:53,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633380) roll requested 2024-12-10T11:03:53,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633387 2024-12-10T11:03:53,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633387 2024-12-10T11:03:53,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633387) roll requested 2024-12-10T11:03:53,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633393 2024-12-10T11:03:53,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633393 2024-12-10T11:03:53,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633393) roll requested 2024-12-10T11:03:53,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633399 2024-12-10T11:03:53,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633399 2024-12-10T11:03:53,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633399) roll requested 2024-12-10T11:03:53,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633406 2024-12-10T11:03:53,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633406 2024-12-10T11:03:53,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633406) roll requested 2024-12-10T11:03:53,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633412 2024-12-10T11:03:53,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633412 2024-12-10T11:03:53,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633412) roll requested 2024-12-10T11:03:53,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633418 2024-12-10T11:03:53,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633418 2024-12-10T11:03:53,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633418) roll requested 2024-12-10T11:03:53,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633425 2024-12-10T11:03:53,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633425 2024-12-10T11:03:53,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633425) roll requested 2024-12-10T11:03:53,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633431 2024-12-10T11:03:53,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633431 2024-12-10T11:03:53,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633431) roll requested 2024-12-10T11:03:53,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633437 2024-12-10T11:03:53,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633437 2024-12-10T11:03:53,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633437) roll requested 2024-12-10T11:03:53,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633443 2024-12-10T11:03:53,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633443 2024-12-10T11:03:53,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633443) roll requested 2024-12-10T11:03:53,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633450 2024-12-10T11:03:53,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633450 2024-12-10T11:03:53,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633450) roll requested 2024-12-10T11:03:53,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633456 2024-12-10T11:03:53,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633456 2024-12-10T11:03:53,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633456) roll requested 2024-12-10T11:03:53,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633462 2024-12-10T11:03:53,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633462 2024-12-10T11:03:53,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633462) roll requested 2024-12-10T11:03:53,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633468 2024-12-10T11:03:53,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633468 2024-12-10T11:03:53,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633468) roll requested 2024-12-10T11:03:53,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633474 2024-12-10T11:03:53,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633474 2024-12-10T11:03:53,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633474) roll requested 2024-12-10T11:03:53,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633481 2024-12-10T11:03:53,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633481 2024-12-10T11:03:53,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633481) roll requested 2024-12-10T11:03:53,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633487 2024-12-10T11:03:53,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633487 2024-12-10T11:03:53,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633487) roll requested 2024-12-10T11:03:53,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633494 2024-12-10T11:03:53,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633494 2024-12-10T11:03:53,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633494) roll requested 2024-12-10T11:03:53,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633500 2024-12-10T11:03:53,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633500 2024-12-10T11:03:53,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633500) roll requested 2024-12-10T11:03:53,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633506 2024-12-10T11:03:53,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633506 2024-12-10T11:03:53,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633506) roll requested 2024-12-10T11:03:53,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633512 2024-12-10T11:03:53,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633512 2024-12-10T11:03:53,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633512) roll requested 2024-12-10T11:03:53,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633518 2024-12-10T11:03:53,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633518 2024-12-10T11:03:53,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633518) roll requested 2024-12-10T11:03:53,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633524 2024-12-10T11:03:53,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633524 2024-12-10T11:03:53,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633524) roll requested 2024-12-10T11:03:53,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633531 2024-12-10T11:03:53,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633531 2024-12-10T11:03:53,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633531) roll requested 2024-12-10T11:03:53,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633536 2024-12-10T11:03:53,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633536 2024-12-10T11:03:53,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633536) roll requested 2024-12-10T11:03:53,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633543 2024-12-10T11:03:53,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633543 2024-12-10T11:03:53,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633543) roll requested 2024-12-10T11:03:53,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633549 2024-12-10T11:03:53,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633549 2024-12-10T11:03:53,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633549) roll requested 2024-12-10T11:03:53,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633555 2024-12-10T11:03:53,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633555 2024-12-10T11:03:53,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633555) roll requested 2024-12-10T11:03:53,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633562 2024-12-10T11:03:53,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633562 2024-12-10T11:03:53,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633562) roll requested 2024-12-10T11:03:53,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633568 2024-12-10T11:03:53,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633568 2024-12-10T11:03:53,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633568) roll requested 2024-12-10T11:03:53,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633574 2024-12-10T11:03:53,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633574 2024-12-10T11:03:53,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633574) roll requested 2024-12-10T11:03:53,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633580 2024-12-10T11:03:53,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633580 2024-12-10T11:03:53,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633580) roll requested 2024-12-10T11:03:53,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633587 2024-12-10T11:03:53,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633587 2024-12-10T11:03:53,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633587) roll requested 2024-12-10T11:03:53,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633593 2024-12-10T11:03:53,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633593 2024-12-10T11:03:53,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633593) roll requested 2024-12-10T11:03:53,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633602 2024-12-10T11:03:53,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633602 2024-12-10T11:03:53,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633602) roll requested 2024-12-10T11:03:53,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633609 2024-12-10T11:03:53,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633609 2024-12-10T11:03:53,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633609) roll requested 2024-12-10T11:03:53,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633615 2024-12-10T11:03:53,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633615 2024-12-10T11:03:53,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633615) roll requested 2024-12-10T11:03:53,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633621 2024-12-10T11:03:53,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633621 2024-12-10T11:03:53,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633621) roll requested 2024-12-10T11:03:53,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633628 2024-12-10T11:03:53,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633628 2024-12-10T11:03:53,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633628) roll requested 2024-12-10T11:03:53,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633638 2024-12-10T11:03:53,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633638 2024-12-10T11:03:53,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633638) roll requested 2024-12-10T11:03:53,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633645 2024-12-10T11:03:53,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633645 2024-12-10T11:03:53,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633645) roll requested 2024-12-10T11:03:53,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633651 2024-12-10T11:03:53,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633651 2024-12-10T11:03:53,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633651) roll requested 2024-12-10T11:03:53,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633657 2024-12-10T11:03:53,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633657 2024-12-10T11:03:53,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633657) roll requested 2024-12-10T11:03:53,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633664 2024-12-10T11:03:53,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633664 2024-12-10T11:03:53,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633664) roll requested 2024-12-10T11:03:53,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633670 2024-12-10T11:03:53,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633670 2024-12-10T11:03:53,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633670) roll requested 2024-12-10T11:03:53,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633676 2024-12-10T11:03:53,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633676 2024-12-10T11:03:53,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633676) roll requested 2024-12-10T11:03:53,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633686 2024-12-10T11:03:53,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633686 2024-12-10T11:03:53,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633686) roll requested 2024-12-10T11:03:53,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633692 2024-12-10T11:03:53,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633692 2024-12-10T11:03:53,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633692) roll requested 2024-12-10T11:03:53,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633698 2024-12-10T11:03:53,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633698 2024-12-10T11:03:53,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633698) roll requested 2024-12-10T11:03:53,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633704 2024-12-10T11:03:53,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633704 2024-12-10T11:03:53,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633704) roll requested 2024-12-10T11:03:53,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633711 2024-12-10T11:03:53,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633711 2024-12-10T11:03:53,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633711) roll requested 2024-12-10T11:03:53,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633717 2024-12-10T11:03:53,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633717 2024-12-10T11:03:53,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633717) roll requested 2024-12-10T11:03:53,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633723 2024-12-10T11:03:53,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633723 2024-12-10T11:03:53,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633723) roll requested 2024-12-10T11:03:53,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633729 2024-12-10T11:03:53,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633729 2024-12-10T11:03:53,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633729) roll requested 2024-12-10T11:03:53,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633735 2024-12-10T11:03:53,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633735 2024-12-10T11:03:53,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633735) roll requested 2024-12-10T11:03:53,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633746 2024-12-10T11:03:53,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633746 2024-12-10T11:03:53,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633746) roll requested 2024-12-10T11:03:53,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633753 2024-12-10T11:03:53,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633753 2024-12-10T11:03:53,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633753) roll requested 2024-12-10T11:03:53,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633761 2024-12-10T11:03:53,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633761 2024-12-10T11:03:53,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633761) roll requested 2024-12-10T11:03:53,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633769 2024-12-10T11:03:53,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633769 2024-12-10T11:03:53,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633769) roll requested 2024-12-10T11:03:53,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633776 2024-12-10T11:03:53,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633776 2024-12-10T11:03:53,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633776) roll requested 2024-12-10T11:03:53,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633783 2024-12-10T11:03:53,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633783 2024-12-10T11:03:53,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633783) roll requested 2024-12-10T11:03:53,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633789 2024-12-10T11:03:53,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633789 2024-12-10T11:03:53,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633789) roll requested 2024-12-10T11:03:53,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633795 2024-12-10T11:03:53,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633795 2024-12-10T11:03:53,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633795) roll requested 2024-12-10T11:03:53,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633802 2024-12-10T11:03:53,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633802 2024-12-10T11:03:53,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633802) roll requested 2024-12-10T11:03:53,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633808 2024-12-10T11:03:53,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633808 2024-12-10T11:03:53,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633808) roll requested 2024-12-10T11:03:53,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633815 2024-12-10T11:03:53,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633815 2024-12-10T11:03:53,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633815) roll requested 2024-12-10T11:03:53,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633821 2024-12-10T11:03:53,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633821 2024-12-10T11:03:53,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633821) roll requested 2024-12-10T11:03:53,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633828 2024-12-10T11:03:53,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633828 2024-12-10T11:03:53,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633828) roll requested 2024-12-10T11:03:53,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633834 2024-12-10T11:03:53,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633834 2024-12-10T11:03:53,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633834) roll requested 2024-12-10T11:03:53,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633840 2024-12-10T11:03:53,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633840 2024-12-10T11:03:53,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633840) roll requested 2024-12-10T11:03:53,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633847 2024-12-10T11:03:53,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633847 2024-12-10T11:03:53,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633847) roll requested 2024-12-10T11:03:53,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633853 2024-12-10T11:03:53,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633853 2024-12-10T11:03:53,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633847 is not closed yet, will try archiving it next time 2024-12-10T11:03:53,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633853) roll requested 2024-12-10T11:03:53,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633859 2024-12-10T11:03:53,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633859 2024-12-10T11:03:53,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633859) roll requested 2024-12-10T11:03:53,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633865 2024-12-10T11:03:53,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633865 2024-12-10T11:03:53,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633865) roll requested 2024-12-10T11:03:53,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633871 2024-12-10T11:03:53,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633871 2024-12-10T11:03:53,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633871) roll requested 2024-12-10T11:03:53,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633878 2024-12-10T11:03:53,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633878 2024-12-10T11:03:53,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633878) roll requested 2024-12-10T11:03:53,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633884 2024-12-10T11:03:53,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633884 2024-12-10T11:03:53,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633884) roll requested 2024-12-10T11:03:53,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633890 2024-12-10T11:03:53,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633890 2024-12-10T11:03:53,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633890) roll requested 2024-12-10T11:03:53,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633897 2024-12-10T11:03:53,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633897 2024-12-10T11:03:53,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633897) roll requested 2024-12-10T11:03:53,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633903 2024-12-10T11:03:53,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633903 2024-12-10T11:03:53,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633903) roll requested 2024-12-10T11:03:53,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633909 2024-12-10T11:03:53,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633909 2024-12-10T11:03:53,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633909) roll requested 2024-12-10T11:03:53,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633915 2024-12-10T11:03:53,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633915 2024-12-10T11:03:53,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633915) roll requested 2024-12-10T11:03:53,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633922 2024-12-10T11:03:53,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633922 2024-12-10T11:03:53,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633922) roll requested 2024-12-10T11:03:53,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633928 2024-12-10T11:03:53,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633928 2024-12-10T11:03:53,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633928) roll requested 2024-12-10T11:03:53,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633934 2024-12-10T11:03:53,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633934 2024-12-10T11:03:53,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633934) roll requested 2024-12-10T11:03:53,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633940 2024-12-10T11:03:53,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633940 2024-12-10T11:03:53,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633940) roll requested 2024-12-10T11:03:53,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633946 2024-12-10T11:03:53,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633946 2024-12-10T11:03:53,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633946) roll requested 2024-12-10T11:03:53,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633952 2024-12-10T11:03:53,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633952 2024-12-10T11:03:53,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633952) roll requested 2024-12-10T11:03:53,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633958 2024-12-10T11:03:53,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633958 2024-12-10T11:03:53,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633958) roll requested 2024-12-10T11:03:53,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633965 2024-12-10T11:03:53,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633965 2024-12-10T11:03:53,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633965) roll requested 2024-12-10T11:03:53,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633971 2024-12-10T11:03:53,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633971 2024-12-10T11:03:53,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633971) roll requested 2024-12-10T11:03:53,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633977 2024-12-10T11:03:53,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633977 2024-12-10T11:03:53,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633977) roll requested 2024-12-10T11:03:53,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633983 2024-12-10T11:03:53,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633983 2024-12-10T11:03:53,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633983) roll requested 2024-12-10T11:03:53,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633991 2024-12-10T11:03:53,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:53,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633991 2024-12-10T11:03:53,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:53,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:53,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633991) roll requested 2024-12-10T11:03:53,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828633998 2024-12-10T11:03:54,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633998 2024-12-10T11:03:54,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828633998) roll requested 2024-12-10T11:03:54,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634004 2024-12-10T11:03:54,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828633998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634004 2024-12-10T11:03:54,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634004) roll requested 2024-12-10T11:03:54,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634010 2024-12-10T11:03:54,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634010 2024-12-10T11:03:54,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634010) roll requested 2024-12-10T11:03:54,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634016 2024-12-10T11:03:54,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634016 2024-12-10T11:03:54,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634016) roll requested 2024-12-10T11:03:54,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634022 2024-12-10T11:03:54,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634022 2024-12-10T11:03:54,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634022) roll requested 2024-12-10T11:03:54,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634028 2024-12-10T11:03:54,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634028 2024-12-10T11:03:54,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634028) roll requested 2024-12-10T11:03:54,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634034 2024-12-10T11:03:54,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634034 2024-12-10T11:03:54,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634034) roll requested 2024-12-10T11:03:54,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634041 2024-12-10T11:03:54,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634041 2024-12-10T11:03:54,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634041) roll requested 2024-12-10T11:03:54,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634047 2024-12-10T11:03:54,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634047 2024-12-10T11:03:54,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634047) roll requested 2024-12-10T11:03:54,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634053 2024-12-10T11:03:54,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634053 2024-12-10T11:03:54,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634053) roll requested 2024-12-10T11:03:54,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634064 2024-12-10T11:03:54,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634064 2024-12-10T11:03:54,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634064) roll requested 2024-12-10T11:03:54,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634070 2024-12-10T11:03:54,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634070 2024-12-10T11:03:54,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634070) roll requested 2024-12-10T11:03:54,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634076 2024-12-10T11:03:54,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634076 2024-12-10T11:03:54,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634076) roll requested 2024-12-10T11:03:54,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634083 2024-12-10T11:03:54,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634083 2024-12-10T11:03:54,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634083) roll requested 2024-12-10T11:03:54,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634089 2024-12-10T11:03:54,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634089 2024-12-10T11:03:54,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634089) roll requested 2024-12-10T11:03:54,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634096 2024-12-10T11:03:54,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634096 2024-12-10T11:03:54,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634096) roll requested 2024-12-10T11:03:54,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634105 2024-12-10T11:03:54,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634105 2024-12-10T11:03:54,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634105) roll requested 2024-12-10T11:03:54,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634112 2024-12-10T11:03:54,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634112 2024-12-10T11:03:54,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634112) roll requested 2024-12-10T11:03:54,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634118 2024-12-10T11:03:54,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634118 2024-12-10T11:03:54,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634118) roll requested 2024-12-10T11:03:54,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634124 2024-12-10T11:03:54,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634124 2024-12-10T11:03:54,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634124) roll requested 2024-12-10T11:03:54,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634130 2024-12-10T11:03:54,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634130 2024-12-10T11:03:54,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634130) roll requested 2024-12-10T11:03:54,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634137 2024-12-10T11:03:54,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634137 2024-12-10T11:03:54,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634137) roll requested 2024-12-10T11:03:54,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634143 2024-12-10T11:03:54,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634143 2024-12-10T11:03:54,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634143) roll requested 2024-12-10T11:03:54,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634149 2024-12-10T11:03:54,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634149 2024-12-10T11:03:54,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634149) roll requested 2024-12-10T11:03:54,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634156 2024-12-10T11:03:54,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634156 2024-12-10T11:03:54,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634156) roll requested 2024-12-10T11:03:54,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634162 2024-12-10T11:03:54,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634162 2024-12-10T11:03:54,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634162) roll requested 2024-12-10T11:03:54,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634168 2024-12-10T11:03:54,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634168 2024-12-10T11:03:54,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634168) roll requested 2024-12-10T11:03:54,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634174 2024-12-10T11:03:54,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634174 2024-12-10T11:03:54,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634174) roll requested 2024-12-10T11:03:54,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634180 2024-12-10T11:03:54,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634180 2024-12-10T11:03:54,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634180) roll requested 2024-12-10T11:03:54,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634190 2024-12-10T11:03:54,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634190 2024-12-10T11:03:54,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634190) roll requested 2024-12-10T11:03:54,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634197 2024-12-10T11:03:54,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634197 2024-12-10T11:03:54,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634197) roll requested 2024-12-10T11:03:54,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634203 2024-12-10T11:03:54,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634203 2024-12-10T11:03:54,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634203) roll requested 2024-12-10T11:03:54,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634209 2024-12-10T11:03:54,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634209 2024-12-10T11:03:54,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634209) roll requested 2024-12-10T11:03:54,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634215 2024-12-10T11:03:54,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634215 2024-12-10T11:03:54,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634215) roll requested 2024-12-10T11:03:54,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634221 2024-12-10T11:03:54,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634221 2024-12-10T11:03:54,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634221) roll requested 2024-12-10T11:03:54,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634228 2024-12-10T11:03:54,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634228 2024-12-10T11:03:54,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634228) roll requested 2024-12-10T11:03:54,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634234 2024-12-10T11:03:54,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634234 2024-12-10T11:03:54,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634234) roll requested 2024-12-10T11:03:54,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634240 2024-12-10T11:03:54,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634240 2024-12-10T11:03:54,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634240) roll requested 2024-12-10T11:03:54,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634246 2024-12-10T11:03:54,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634246 2024-12-10T11:03:54,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634246) roll requested 2024-12-10T11:03:54,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634252 2024-12-10T11:03:54,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634252 2024-12-10T11:03:54,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634252) roll requested 2024-12-10T11:03:54,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634260 2024-12-10T11:03:54,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634260 2024-12-10T11:03:54,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634260) roll requested 2024-12-10T11:03:54,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634266 2024-12-10T11:03:54,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634266 2024-12-10T11:03:54,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634266) roll requested 2024-12-10T11:03:54,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634272 2024-12-10T11:03:54,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634272 2024-12-10T11:03:54,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634272) roll requested 2024-12-10T11:03:54,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634278 2024-12-10T11:03:54,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634278 2024-12-10T11:03:54,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634278) roll requested 2024-12-10T11:03:54,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634284 2024-12-10T11:03:54,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634284 2024-12-10T11:03:54,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634284) roll requested 2024-12-10T11:03:54,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634290 2024-12-10T11:03:54,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634290 2024-12-10T11:03:54,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634290) roll requested 2024-12-10T11:03:54,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634296 2024-12-10T11:03:54,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634296 2024-12-10T11:03:54,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634296) roll requested 2024-12-10T11:03:54,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634304 2024-12-10T11:03:54,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634304 2024-12-10T11:03:54,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634304) roll requested 2024-12-10T11:03:54,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634310 2024-12-10T11:03:54,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634310 2024-12-10T11:03:54,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634310) roll requested 2024-12-10T11:03:54,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634316 2024-12-10T11:03:54,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634316 2024-12-10T11:03:54,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634316) roll requested 2024-12-10T11:03:54,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634322 2024-12-10T11:03:54,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634322 2024-12-10T11:03:54,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634322) roll requested 2024-12-10T11:03:54,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634329 2024-12-10T11:03:54,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634329 2024-12-10T11:03:54,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634329) roll requested 2024-12-10T11:03:54,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634343 2024-12-10T11:03:54,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634343 2024-12-10T11:03:54,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634343) roll requested 2024-12-10T11:03:54,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634352 2024-12-10T11:03:54,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634352 2024-12-10T11:03:54,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634352) roll requested 2024-12-10T11:03:54,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634360 2024-12-10T11:03:54,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634360 2024-12-10T11:03:54,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634360) roll requested 2024-12-10T11:03:54,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634368 2024-12-10T11:03:54,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634368 2024-12-10T11:03:54,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634368) roll requested 2024-12-10T11:03:54,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634376 2024-12-10T11:03:54,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634376 2024-12-10T11:03:54,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634376) roll requested 2024-12-10T11:03:54,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634384 2024-12-10T11:03:54,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634384 2024-12-10T11:03:54,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634384) roll requested 2024-12-10T11:03:54,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634391 2024-12-10T11:03:54,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634391 2024-12-10T11:03:54,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634391) roll requested 2024-12-10T11:03:54,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634397 2024-12-10T11:03:54,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634397 2024-12-10T11:03:54,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634397) roll requested 2024-12-10T11:03:54,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634403 2024-12-10T11:03:54,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634403 2024-12-10T11:03:54,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634403) roll requested 2024-12-10T11:03:54,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634409 2024-12-10T11:03:54,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634409 2024-12-10T11:03:54,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634409) roll requested 2024-12-10T11:03:54,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634415 2024-12-10T11:03:54,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634415 2024-12-10T11:03:54,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634415) roll requested 2024-12-10T11:03:54,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634421 2024-12-10T11:03:54,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634421 2024-12-10T11:03:54,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634421) roll requested 2024-12-10T11:03:54,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634428 2024-12-10T11:03:54,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634428 2024-12-10T11:03:54,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634428) roll requested 2024-12-10T11:03:54,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634434 2024-12-10T11:03:54,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634434 2024-12-10T11:03:54,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634434) roll requested 2024-12-10T11:03:54,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634440 2024-12-10T11:03:54,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634440 2024-12-10T11:03:54,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634440) roll requested 2024-12-10T11:03:54,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634446 2024-12-10T11:03:54,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634446 2024-12-10T11:03:54,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634446) roll requested 2024-12-10T11:03:54,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634452 2024-12-10T11:03:54,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634452 2024-12-10T11:03:54,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634452) roll requested 2024-12-10T11:03:54,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634458 2024-12-10T11:03:54,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634458 2024-12-10T11:03:54,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634458) roll requested 2024-12-10T11:03:54,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634469 2024-12-10T11:03:54,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634469 2024-12-10T11:03:54,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634469) roll requested 2024-12-10T11:03:54,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634475 2024-12-10T11:03:54,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634475 2024-12-10T11:03:54,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634475) roll requested 2024-12-10T11:03:54,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634481 2024-12-10T11:03:54,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634481 2024-12-10T11:03:54,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634481) roll requested 2024-12-10T11:03:54,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634487 2024-12-10T11:03:54,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634487 2024-12-10T11:03:54,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634487) roll requested 2024-12-10T11:03:54,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634494 2024-12-10T11:03:54,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634494 2024-12-10T11:03:54,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634494) roll requested 2024-12-10T11:03:54,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634500 2024-12-10T11:03:54,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634500 2024-12-10T11:03:54,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634500) roll requested 2024-12-10T11:03:54,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634506 2024-12-10T11:03:54,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634506 2024-12-10T11:03:54,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634506) roll requested 2024-12-10T11:03:54,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634512 2024-12-10T11:03:54,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634512 2024-12-10T11:03:54,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634512) roll requested 2024-12-10T11:03:54,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634518 2024-12-10T11:03:54,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634518 2024-12-10T11:03:54,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634518) roll requested 2024-12-10T11:03:54,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634524 2024-12-10T11:03:54,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634524 2024-12-10T11:03:54,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634524) roll requested 2024-12-10T11:03:54,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634530 2024-12-10T11:03:54,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634530 2024-12-10T11:03:54,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634530) roll requested 2024-12-10T11:03:54,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634536 2024-12-10T11:03:54,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634536 2024-12-10T11:03:54,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634536) roll requested 2024-12-10T11:03:54,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634542 2024-12-10T11:03:54,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634542 2024-12-10T11:03:54,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634542) roll requested 2024-12-10T11:03:54,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634549 2024-12-10T11:03:54,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634549 2024-12-10T11:03:54,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634549) roll requested 2024-12-10T11:03:54,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634555 2024-12-10T11:03:54,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634555 2024-12-10T11:03:54,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634555) roll requested 2024-12-10T11:03:54,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634562 2024-12-10T11:03:54,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634562 2024-12-10T11:03:54,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634562) roll requested 2024-12-10T11:03:54,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634568 2024-12-10T11:03:54,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634568 2024-12-10T11:03:54,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634568) roll requested 2024-12-10T11:03:54,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634574 2024-12-10T11:03:54,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634574 2024-12-10T11:03:54,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634574) roll requested 2024-12-10T11:03:54,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634580 2024-12-10T11:03:54,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634580 2024-12-10T11:03:54,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634580) roll requested 2024-12-10T11:03:54,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634586 2024-12-10T11:03:54,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634586 2024-12-10T11:03:54,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634586) roll requested 2024-12-10T11:03:54,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634592 2024-12-10T11:03:54,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634592 2024-12-10T11:03:54,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634592) roll requested 2024-12-10T11:03:54,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634598 2024-12-10T11:03:54,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634598 2024-12-10T11:03:54,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634598) roll requested 2024-12-10T11:03:54,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634605 2024-12-10T11:03:54,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634605 2024-12-10T11:03:54,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634605) roll requested 2024-12-10T11:03:54,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634611 2024-12-10T11:03:54,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634611 2024-12-10T11:03:54,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634611) roll requested 2024-12-10T11:03:54,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634617 2024-12-10T11:03:54,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634617 2024-12-10T11:03:54,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634617) roll requested 2024-12-10T11:03:54,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634623 2024-12-10T11:03:54,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634623 2024-12-10T11:03:54,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634623) roll requested 2024-12-10T11:03:54,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634630 2024-12-10T11:03:54,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634630 2024-12-10T11:03:54,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634630) roll requested 2024-12-10T11:03:54,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634641 2024-12-10T11:03:54,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634641 2024-12-10T11:03:54,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634641) roll requested 2024-12-10T11:03:54,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634647 2024-12-10T11:03:54,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634647 2024-12-10T11:03:54,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634647) roll requested 2024-12-10T11:03:54,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634653 2024-12-10T11:03:54,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634653 2024-12-10T11:03:54,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634653) roll requested 2024-12-10T11:03:54,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634660 2024-12-10T11:03:54,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634660 2024-12-10T11:03:54,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634660) roll requested 2024-12-10T11:03:54,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634669 2024-12-10T11:03:54,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634669 2024-12-10T11:03:54,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634669) roll requested 2024-12-10T11:03:54,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634676 2024-12-10T11:03:54,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634676 2024-12-10T11:03:54,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634676) roll requested 2024-12-10T11:03:54,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634682 2024-12-10T11:03:54,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634682 2024-12-10T11:03:54,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634682) roll requested 2024-12-10T11:03:54,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634688 2024-12-10T11:03:54,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634688 2024-12-10T11:03:54,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634688) roll requested 2024-12-10T11:03:54,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634694 2024-12-10T11:03:54,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634694 2024-12-10T11:03:54,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634694) roll requested 2024-12-10T11:03:54,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634700 2024-12-10T11:03:54,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634700 2024-12-10T11:03:54,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634700) roll requested 2024-12-10T11:03:54,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634707 2024-12-10T11:03:54,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634707 2024-12-10T11:03:54,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634707) roll requested 2024-12-10T11:03:54,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634713 2024-12-10T11:03:54,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634713 2024-12-10T11:03:54,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634713) roll requested 2024-12-10T11:03:54,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634719 2024-12-10T11:03:54,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634719 2024-12-10T11:03:54,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634719) roll requested 2024-12-10T11:03:54,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634726 2024-12-10T11:03:54,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634726 2024-12-10T11:03:54,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634726) roll requested 2024-12-10T11:03:54,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634732 2024-12-10T11:03:54,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634732 2024-12-10T11:03:54,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634732) roll requested 2024-12-10T11:03:54,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634738 2024-12-10T11:03:54,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634738 2024-12-10T11:03:54,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634738) roll requested 2024-12-10T11:03:54,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634744 2024-12-10T11:03:54,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634744 2024-12-10T11:03:54,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634744) roll requested 2024-12-10T11:03:54,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634751 2024-12-10T11:03:54,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634751 2024-12-10T11:03:54,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634751) roll requested 2024-12-10T11:03:54,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634757 2024-12-10T11:03:54,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634757 2024-12-10T11:03:54,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634757) roll requested 2024-12-10T11:03:54,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634763 2024-12-10T11:03:54,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634763 2024-12-10T11:03:54,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634763) roll requested 2024-12-10T11:03:54,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634769 2024-12-10T11:03:54,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634769 2024-12-10T11:03:54,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634769) roll requested 2024-12-10T11:03:54,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634775 2024-12-10T11:03:54,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634775 2024-12-10T11:03:54,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634775) roll requested 2024-12-10T11:03:54,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634782 2024-12-10T11:03:54,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634782 2024-12-10T11:03:54,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634782) roll requested 2024-12-10T11:03:54,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634788 2024-12-10T11:03:54,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634788 2024-12-10T11:03:54,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634788) roll requested 2024-12-10T11:03:54,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634794 2024-12-10T11:03:54,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634794 2024-12-10T11:03:54,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634794) roll requested 2024-12-10T11:03:54,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634801 2024-12-10T11:03:54,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634801 2024-12-10T11:03:54,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634801) roll requested 2024-12-10T11:03:54,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634807 2024-12-10T11:03:54,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634807 2024-12-10T11:03:54,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634807) roll requested 2024-12-10T11:03:54,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634813 2024-12-10T11:03:54,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634813 2024-12-10T11:03:54,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634813) roll requested 2024-12-10T11:03:54,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634820 2024-12-10T11:03:54,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634820 2024-12-10T11:03:54,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634820) roll requested 2024-12-10T11:03:54,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634826 2024-12-10T11:03:54,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634826 2024-12-10T11:03:54,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634826) roll requested 2024-12-10T11:03:54,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634834 2024-12-10T11:03:54,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634834 2024-12-10T11:03:54,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634834) roll requested 2024-12-10T11:03:54,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634840 2024-12-10T11:03:54,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634840 2024-12-10T11:03:54,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634840) roll requested 2024-12-10T11:03:54,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634846 2024-12-10T11:03:54,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634846 2024-12-10T11:03:54,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634846) roll requested 2024-12-10T11:03:54,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634852 2024-12-10T11:03:54,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634852 2024-12-10T11:03:54,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634852) roll requested 2024-12-10T11:03:54,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634858 2024-12-10T11:03:54,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634858 2024-12-10T11:03:54,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634858) roll requested 2024-12-10T11:03:54,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634865 2024-12-10T11:03:54,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634865 2024-12-10T11:03:54,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634865) roll requested 2024-12-10T11:03:54,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634871 2024-12-10T11:03:54,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634871 2024-12-10T11:03:54,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634871) roll requested 2024-12-10T11:03:54,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634877 2024-12-10T11:03:54,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634877 2024-12-10T11:03:54,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634877) roll requested 2024-12-10T11:03:54,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634883 2024-12-10T11:03:54,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634883 2024-12-10T11:03:54,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634883) roll requested 2024-12-10T11:03:54,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634890 2024-12-10T11:03:54,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634890 2024-12-10T11:03:54,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634890) roll requested 2024-12-10T11:03:54,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634896 2024-12-10T11:03:54,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634896 2024-12-10T11:03:54,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634896) roll requested 2024-12-10T11:03:54,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634902 2024-12-10T11:03:54,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634902 2024-12-10T11:03:54,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634902) roll requested 2024-12-10T11:03:54,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634908 2024-12-10T11:03:54,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634908 2024-12-10T11:03:54,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634908) roll requested 2024-12-10T11:03:54,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634915 2024-12-10T11:03:54,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634915 2024-12-10T11:03:54,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634915) roll requested 2024-12-10T11:03:54,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634921 2024-12-10T11:03:54,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634921 2024-12-10T11:03:54,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634921) roll requested 2024-12-10T11:03:54,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634927 2024-12-10T11:03:54,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634927 2024-12-10T11:03:54,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634927) roll requested 2024-12-10T11:03:54,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634934 2024-12-10T11:03:54,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634934 2024-12-10T11:03:54,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634934) roll requested 2024-12-10T11:03:54,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634944 2024-12-10T11:03:54,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634944 2024-12-10T11:03:54,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634944) roll requested 2024-12-10T11:03:54,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634951 2024-12-10T11:03:54,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634951 2024-12-10T11:03:54,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634951) roll requested 2024-12-10T11:03:54,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634957 2024-12-10T11:03:54,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634957 2024-12-10T11:03:54,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634957) roll requested 2024-12-10T11:03:54,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634963 2024-12-10T11:03:54,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634963 2024-12-10T11:03:54,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634963) roll requested 2024-12-10T11:03:54,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634970 2024-12-10T11:03:54,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634970 2024-12-10T11:03:54,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634970) roll requested 2024-12-10T11:03:54,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634976 2024-12-10T11:03:54,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634976 2024-12-10T11:03:54,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634976) roll requested 2024-12-10T11:03:54,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634982 2024-12-10T11:03:54,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634982 2024-12-10T11:03:54,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634982) roll requested 2024-12-10T11:03:54,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634988 2024-12-10T11:03:54,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:54,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634988 2024-12-10T11:03:54,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:54,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:54,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634988) roll requested 2024-12-10T11:03:54,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828634995 2024-12-10T11:03:55,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634995 2024-12-10T11:03:55,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828634995) roll requested 2024-12-10T11:03:55,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635002 2024-12-10T11:03:55,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828634995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635002 2024-12-10T11:03:55,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635002) roll requested 2024-12-10T11:03:55,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635008 2024-12-10T11:03:55,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635008 2024-12-10T11:03:55,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635008) roll requested 2024-12-10T11:03:55,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635014 2024-12-10T11:03:55,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635014 2024-12-10T11:03:55,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635014) roll requested 2024-12-10T11:03:55,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635020 2024-12-10T11:03:55,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635020 2024-12-10T11:03:55,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635020) roll requested 2024-12-10T11:03:55,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635026 2024-12-10T11:03:55,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635026 2024-12-10T11:03:55,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635026) roll requested 2024-12-10T11:03:55,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635032 2024-12-10T11:03:55,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635032 2024-12-10T11:03:55,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635032) roll requested 2024-12-10T11:03:55,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635039 2024-12-10T11:03:55,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635039 2024-12-10T11:03:55,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635039) roll requested 2024-12-10T11:03:55,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635045 2024-12-10T11:03:55,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635045 2024-12-10T11:03:55,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635045) roll requested 2024-12-10T11:03:55,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635052 2024-12-10T11:03:55,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635052 2024-12-10T11:03:55,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635052) roll requested 2024-12-10T11:03:55,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635058 2024-12-10T11:03:55,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635058 2024-12-10T11:03:55,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635058) roll requested 2024-12-10T11:03:55,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635064 2024-12-10T11:03:55,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635064 2024-12-10T11:03:55,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635064) roll requested 2024-12-10T11:03:55,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635071 2024-12-10T11:03:55,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635071 2024-12-10T11:03:55,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635071) roll requested 2024-12-10T11:03:55,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635077 2024-12-10T11:03:55,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635077 2024-12-10T11:03:55,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635077) roll requested 2024-12-10T11:03:55,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635083 2024-12-10T11:03:55,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635083 2024-12-10T11:03:55,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635083) roll requested 2024-12-10T11:03:55,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635090 2024-12-10T11:03:55,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635090 2024-12-10T11:03:55,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635090) roll requested 2024-12-10T11:03:55,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635096 2024-12-10T11:03:55,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635096 2024-12-10T11:03:55,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635096) roll requested 2024-12-10T11:03:55,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635103 2024-12-10T11:03:55,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635103 2024-12-10T11:03:55,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635103) roll requested 2024-12-10T11:03:55,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635109 2024-12-10T11:03:55,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635109 2024-12-10T11:03:55,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635109) roll requested 2024-12-10T11:03:55,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635115 2024-12-10T11:03:55,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635115 2024-12-10T11:03:55,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635115) roll requested 2024-12-10T11:03:55,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635122 2024-12-10T11:03:55,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635122 2024-12-10T11:03:55,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635122) roll requested 2024-12-10T11:03:55,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635128 2024-12-10T11:03:55,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635128 2024-12-10T11:03:55,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635128) roll requested 2024-12-10T11:03:55,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635134 2024-12-10T11:03:55,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635134 2024-12-10T11:03:55,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635134) roll requested 2024-12-10T11:03:55,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635141 2024-12-10T11:03:55,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635141 2024-12-10T11:03:55,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635141) roll requested 2024-12-10T11:03:55,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635153 2024-12-10T11:03:55,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635153 2024-12-10T11:03:55,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635153) roll requested 2024-12-10T11:03:55,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635161 2024-12-10T11:03:55,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635161 2024-12-10T11:03:55,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635161) roll requested 2024-12-10T11:03:55,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635170 2024-12-10T11:03:55,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635170 2024-12-10T11:03:55,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635170) roll requested 2024-12-10T11:03:55,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635177 2024-12-10T11:03:55,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635177 2024-12-10T11:03:55,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635177) roll requested 2024-12-10T11:03:55,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635185 2024-12-10T11:03:55,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635185 2024-12-10T11:03:55,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635185) roll requested 2024-12-10T11:03:55,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635191 2024-12-10T11:03:55,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635191 2024-12-10T11:03:55,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635191) roll requested 2024-12-10T11:03:55,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635198 2024-12-10T11:03:55,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635198 2024-12-10T11:03:55,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635198) roll requested 2024-12-10T11:03:55,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635204 2024-12-10T11:03:55,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635204 2024-12-10T11:03:55,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635204) roll requested 2024-12-10T11:03:55,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635210 2024-12-10T11:03:55,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635210 2024-12-10T11:03:55,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635210) roll requested 2024-12-10T11:03:55,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635216 2024-12-10T11:03:55,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635216 2024-12-10T11:03:55,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635216) roll requested 2024-12-10T11:03:55,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635222 2024-12-10T11:03:55,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635222 2024-12-10T11:03:55,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635222) roll requested 2024-12-10T11:03:55,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635228 2024-12-10T11:03:55,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635228 2024-12-10T11:03:55,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635228) roll requested 2024-12-10T11:03:55,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635234 2024-12-10T11:03:55,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635234 2024-12-10T11:03:55,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635234) roll requested 2024-12-10T11:03:55,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635241 2024-12-10T11:03:55,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635241 2024-12-10T11:03:55,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635241) roll requested 2024-12-10T11:03:55,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635247 2024-12-10T11:03:55,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635247 2024-12-10T11:03:55,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635247) roll requested 2024-12-10T11:03:55,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635253 2024-12-10T11:03:55,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635253 2024-12-10T11:03:55,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635253) roll requested 2024-12-10T11:03:55,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635259 2024-12-10T11:03:55,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635259 2024-12-10T11:03:55,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635259) roll requested 2024-12-10T11:03:55,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635265 2024-12-10T11:03:55,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635265 2024-12-10T11:03:55,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635265) roll requested 2024-12-10T11:03:55,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635271 2024-12-10T11:03:55,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635271 2024-12-10T11:03:55,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635271) roll requested 2024-12-10T11:03:55,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635278 2024-12-10T11:03:55,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635278 2024-12-10T11:03:55,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635278) roll requested 2024-12-10T11:03:55,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635284 2024-12-10T11:03:55,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635284 2024-12-10T11:03:55,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635284) roll requested 2024-12-10T11:03:55,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635291 2024-12-10T11:03:55,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635291 2024-12-10T11:03:55,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635291) roll requested 2024-12-10T11:03:55,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635297 2024-12-10T11:03:55,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635297 2024-12-10T11:03:55,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635297) roll requested 2024-12-10T11:03:55,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635303 2024-12-10T11:03:55,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635303 2024-12-10T11:03:55,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635303) roll requested 2024-12-10T11:03:55,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635309 2024-12-10T11:03:55,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635309 2024-12-10T11:03:55,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635309) roll requested 2024-12-10T11:03:55,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635315 2024-12-10T11:03:55,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635315 2024-12-10T11:03:55,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635315) roll requested 2024-12-10T11:03:55,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635321 2024-12-10T11:03:55,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635321 2024-12-10T11:03:55,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635321) roll requested 2024-12-10T11:03:55,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635328 2024-12-10T11:03:55,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635328 2024-12-10T11:03:55,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635328) roll requested 2024-12-10T11:03:55,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635334 2024-12-10T11:03:55,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635334 2024-12-10T11:03:55,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635334) roll requested 2024-12-10T11:03:55,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635340 2024-12-10T11:03:55,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635340 2024-12-10T11:03:55,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635340) roll requested 2024-12-10T11:03:55,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635346 2024-12-10T11:03:55,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635346 2024-12-10T11:03:55,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635346) roll requested 2024-12-10T11:03:55,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635352 2024-12-10T11:03:55,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635352 2024-12-10T11:03:55,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635352) roll requested 2024-12-10T11:03:55,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635358 2024-12-10T11:03:55,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635358 2024-12-10T11:03:55,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635358) roll requested 2024-12-10T11:03:55,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635364 2024-12-10T11:03:55,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635364 2024-12-10T11:03:55,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635364) roll requested 2024-12-10T11:03:55,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635371 2024-12-10T11:03:55,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635371 2024-12-10T11:03:55,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635371) roll requested 2024-12-10T11:03:55,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635377 2024-12-10T11:03:55,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635377 2024-12-10T11:03:55,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635377) roll requested 2024-12-10T11:03:55,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635383 2024-12-10T11:03:55,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635383 2024-12-10T11:03:55,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635383) roll requested 2024-12-10T11:03:55,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635389 2024-12-10T11:03:55,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635389 2024-12-10T11:03:55,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635389) roll requested 2024-12-10T11:03:55,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635395 2024-12-10T11:03:55,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635395 2024-12-10T11:03:55,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635395) roll requested 2024-12-10T11:03:55,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635403 2024-12-10T11:03:55,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635403 2024-12-10T11:03:55,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635403) roll requested 2024-12-10T11:03:55,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635409 2024-12-10T11:03:55,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635409 2024-12-10T11:03:55,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635409) roll requested 2024-12-10T11:03:55,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635415 2024-12-10T11:03:55,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635415 2024-12-10T11:03:55,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635415) roll requested 2024-12-10T11:03:55,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635421 2024-12-10T11:03:55,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635421 2024-12-10T11:03:55,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635421) roll requested 2024-12-10T11:03:55,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635427 2024-12-10T11:03:55,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635427 2024-12-10T11:03:55,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635427) roll requested 2024-12-10T11:03:55,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635433 2024-12-10T11:03:55,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635433 2024-12-10T11:03:55,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635433) roll requested 2024-12-10T11:03:55,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635440 2024-12-10T11:03:55,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635440 2024-12-10T11:03:55,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635440) roll requested 2024-12-10T11:03:55,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635446 2024-12-10T11:03:55,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635446 2024-12-10T11:03:55,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635446) roll requested 2024-12-10T11:03:55,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635453 2024-12-10T11:03:55,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635453 2024-12-10T11:03:55,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635453) roll requested 2024-12-10T11:03:55,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635459 2024-12-10T11:03:55,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635459 2024-12-10T11:03:55,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635459) roll requested 2024-12-10T11:03:55,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635465 2024-12-10T11:03:55,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635465 2024-12-10T11:03:55,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635465) roll requested 2024-12-10T11:03:55,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635471 2024-12-10T11:03:55,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635471 2024-12-10T11:03:55,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635471) roll requested 2024-12-10T11:03:55,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635478 2024-12-10T11:03:55,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635478 2024-12-10T11:03:55,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635478) roll requested 2024-12-10T11:03:55,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635484 2024-12-10T11:03:55,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635484 2024-12-10T11:03:55,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635484) roll requested 2024-12-10T11:03:55,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635490 2024-12-10T11:03:55,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635490 2024-12-10T11:03:55,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635490) roll requested 2024-12-10T11:03:55,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635496 2024-12-10T11:03:55,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635496 2024-12-10T11:03:55,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635496) roll requested 2024-12-10T11:03:55,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635502 2024-12-10T11:03:55,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635502 2024-12-10T11:03:55,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635502) roll requested 2024-12-10T11:03:55,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635508 2024-12-10T11:03:55,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635508 2024-12-10T11:03:55,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635508) roll requested 2024-12-10T11:03:55,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635514 2024-12-10T11:03:55,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635514 2024-12-10T11:03:55,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635514) roll requested 2024-12-10T11:03:55,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635521 2024-12-10T11:03:55,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635521 2024-12-10T11:03:55,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635521) roll requested 2024-12-10T11:03:55,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635532 2024-12-10T11:03:55,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635532 2024-12-10T11:03:55,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635532) roll requested 2024-12-10T11:03:55,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635538 2024-12-10T11:03:55,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635538 2024-12-10T11:03:55,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635538) roll requested 2024-12-10T11:03:55,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635548 2024-12-10T11:03:55,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635548 2024-12-10T11:03:55,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635548) roll requested 2024-12-10T11:03:55,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635554 2024-12-10T11:03:55,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635554 2024-12-10T11:03:55,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635554) roll requested 2024-12-10T11:03:55,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635561 2024-12-10T11:03:55,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635561 2024-12-10T11:03:55,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635561) roll requested 2024-12-10T11:03:55,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635567 2024-12-10T11:03:55,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635567 2024-12-10T11:03:55,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635567) roll requested 2024-12-10T11:03:55,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635573 2024-12-10T11:03:55,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635573 2024-12-10T11:03:55,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635573) roll requested 2024-12-10T11:03:55,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635579 2024-12-10T11:03:55,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635579 2024-12-10T11:03:55,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635579) roll requested 2024-12-10T11:03:55,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635585 2024-12-10T11:03:55,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635585 2024-12-10T11:03:55,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635585) roll requested 2024-12-10T11:03:55,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635591 2024-12-10T11:03:55,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635591 2024-12-10T11:03:55,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635591) roll requested 2024-12-10T11:03:55,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635598 2024-12-10T11:03:55,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635598 2024-12-10T11:03:55,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635598) roll requested 2024-12-10T11:03:55,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635605 2024-12-10T11:03:55,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635605 2024-12-10T11:03:55,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635605) roll requested 2024-12-10T11:03:55,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635612 2024-12-10T11:03:55,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635612 2024-12-10T11:03:55,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635612) roll requested 2024-12-10T11:03:55,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635618 2024-12-10T11:03:55,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635618 2024-12-10T11:03:55,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635618) roll requested 2024-12-10T11:03:55,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635624 2024-12-10T11:03:55,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635624 2024-12-10T11:03:55,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635624) roll requested 2024-12-10T11:03:55,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635635 2024-12-10T11:03:55,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635635 2024-12-10T11:03:55,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635635) roll requested 2024-12-10T11:03:55,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635641 2024-12-10T11:03:55,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635641 2024-12-10T11:03:55,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635641) roll requested 2024-12-10T11:03:55,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635648 2024-12-10T11:03:55,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635648 2024-12-10T11:03:55,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635648) roll requested 2024-12-10T11:03:55,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635654 2024-12-10T11:03:55,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635654 2024-12-10T11:03:55,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635654) roll requested 2024-12-10T11:03:55,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635660 2024-12-10T11:03:55,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635660 2024-12-10T11:03:55,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635660) roll requested 2024-12-10T11:03:55,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635669 2024-12-10T11:03:55,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635669 2024-12-10T11:03:55,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635669) roll requested 2024-12-10T11:03:55,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635677 2024-12-10T11:03:55,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635677 2024-12-10T11:03:55,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635677) roll requested 2024-12-10T11:03:55,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635685 2024-12-10T11:03:55,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635685 2024-12-10T11:03:55,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635685) roll requested 2024-12-10T11:03:55,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635691 2024-12-10T11:03:55,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635691 2024-12-10T11:03:55,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635691) roll requested 2024-12-10T11:03:55,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635698 2024-12-10T11:03:55,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635698 2024-12-10T11:03:55,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635698) roll requested 2024-12-10T11:03:55,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635704 2024-12-10T11:03:55,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635704 2024-12-10T11:03:55,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635704) roll requested 2024-12-10T11:03:55,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635710 2024-12-10T11:03:55,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635710 2024-12-10T11:03:55,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635710) roll requested 2024-12-10T11:03:55,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635716 2024-12-10T11:03:55,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635716 2024-12-10T11:03:55,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635716) roll requested 2024-12-10T11:03:55,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635722 2024-12-10T11:03:55,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635722 2024-12-10T11:03:55,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635722) roll requested 2024-12-10T11:03:55,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635728 2024-12-10T11:03:55,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635728 2024-12-10T11:03:55,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635728) roll requested 2024-12-10T11:03:55,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635734 2024-12-10T11:03:55,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635734 2024-12-10T11:03:55,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635734) roll requested 2024-12-10T11:03:55,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635741 2024-12-10T11:03:55,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635741 2024-12-10T11:03:55,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635741) roll requested 2024-12-10T11:03:55,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635747 2024-12-10T11:03:55,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635747 2024-12-10T11:03:55,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635747) roll requested 2024-12-10T11:03:55,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635753 2024-12-10T11:03:55,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635753 2024-12-10T11:03:55,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635753) roll requested 2024-12-10T11:03:55,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635762 2024-12-10T11:03:55,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635762 2024-12-10T11:03:55,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635762) roll requested 2024-12-10T11:03:55,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635772 2024-12-10T11:03:55,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635772 2024-12-10T11:03:55,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635772) roll requested 2024-12-10T11:03:55,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635778 2024-12-10T11:03:55,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635778 2024-12-10T11:03:55,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635778) roll requested 2024-12-10T11:03:55,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635784 2024-12-10T11:03:55,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635784 2024-12-10T11:03:55,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635784) roll requested 2024-12-10T11:03:55,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635793 2024-12-10T11:03:55,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635793 2024-12-10T11:03:55,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635793) roll requested 2024-12-10T11:03:55,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635799 2024-12-10T11:03:55,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635799 2024-12-10T11:03:55,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635799) roll requested 2024-12-10T11:03:55,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635806 2024-12-10T11:03:55,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635806 2024-12-10T11:03:55,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635806) roll requested 2024-12-10T11:03:55,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635813 2024-12-10T11:03:55,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635813 2024-12-10T11:03:55,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635813) roll requested 2024-12-10T11:03:55,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635820 2024-12-10T11:03:55,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635820 2024-12-10T11:03:55,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635820) roll requested 2024-12-10T11:03:55,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635829 2024-12-10T11:03:55,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635829 2024-12-10T11:03:55,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635829) roll requested 2024-12-10T11:03:55,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635836 2024-12-10T11:03:55,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635836 2024-12-10T11:03:55,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635836) roll requested 2024-12-10T11:03:55,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635842 2024-12-10T11:03:55,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635842 2024-12-10T11:03:55,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635842) roll requested 2024-12-10T11:03:55,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635849 2024-12-10T11:03:55,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635849 2024-12-10T11:03:55,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635849) roll requested 2024-12-10T11:03:55,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635857 2024-12-10T11:03:55,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635857 2024-12-10T11:03:55,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635857) roll requested 2024-12-10T11:03:55,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635865 2024-12-10T11:03:55,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635865 2024-12-10T11:03:55,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635865) roll requested 2024-12-10T11:03:55,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635872 2024-12-10T11:03:55,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635872 2024-12-10T11:03:55,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635872) roll requested 2024-12-10T11:03:55,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635878 2024-12-10T11:03:55,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635878 2024-12-10T11:03:55,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635878) roll requested 2024-12-10T11:03:55,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635884 2024-12-10T11:03:55,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635884 2024-12-10T11:03:55,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635884) roll requested 2024-12-10T11:03:55,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635891 2024-12-10T11:03:55,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635891 2024-12-10T11:03:55,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635891) roll requested 2024-12-10T11:03:55,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635899 2024-12-10T11:03:55,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635899 2024-12-10T11:03:55,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635899) roll requested 2024-12-10T11:03:55,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635905 2024-12-10T11:03:55,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635905 2024-12-10T11:03:55,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635905) roll requested 2024-12-10T11:03:55,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635911 2024-12-10T11:03:55,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635911 2024-12-10T11:03:55,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635911) roll requested 2024-12-10T11:03:55,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635917 2024-12-10T11:03:55,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635917 2024-12-10T11:03:55,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635917) roll requested 2024-12-10T11:03:55,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635924 2024-12-10T11:03:55,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635924 2024-12-10T11:03:55,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635924) roll requested 2024-12-10T11:03:55,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635930 2024-12-10T11:03:55,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635930 2024-12-10T11:03:55,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635930) roll requested 2024-12-10T11:03:55,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635936 2024-12-10T11:03:55,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635936 2024-12-10T11:03:55,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635936) roll requested 2024-12-10T11:03:55,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635942 2024-12-10T11:03:55,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635942 2024-12-10T11:03:55,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635942) roll requested 2024-12-10T11:03:55,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635949 2024-12-10T11:03:55,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635949 2024-12-10T11:03:55,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635949) roll requested 2024-12-10T11:03:55,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635956 2024-12-10T11:03:55,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635956 2024-12-10T11:03:55,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635956) roll requested 2024-12-10T11:03:55,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635962 2024-12-10T11:03:55,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635962 2024-12-10T11:03:55,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635962) roll requested 2024-12-10T11:03:55,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635968 2024-12-10T11:03:55,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635968 2024-12-10T11:03:55,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635968) roll requested 2024-12-10T11:03:55,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635974 2024-12-10T11:03:55,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635974 2024-12-10T11:03:55,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635974) roll requested 2024-12-10T11:03:55,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635980 2024-12-10T11:03:55,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635980 2024-12-10T11:03:55,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635980) roll requested 2024-12-10T11:03:55,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635987 2024-12-10T11:03:55,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635987 2024-12-10T11:03:55,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:55,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:55,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635987) roll requested 2024-12-10T11:03:55,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828635994 2024-12-10T11:03:55,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:55,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635994 2024-12-10T11:03:56,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828635994) roll requested 2024-12-10T11:03:56,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636000 2024-12-10T11:03:56,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828635994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636000 2024-12-10T11:03:56,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636000) roll requested 2024-12-10T11:03:56,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636007 2024-12-10T11:03:56,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636007 2024-12-10T11:03:56,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636007) roll requested 2024-12-10T11:03:56,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636013 2024-12-10T11:03:56,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636013 2024-12-10T11:03:56,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636013) roll requested 2024-12-10T11:03:56,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636019 2024-12-10T11:03:56,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636019 2024-12-10T11:03:56,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636019) roll requested 2024-12-10T11:03:56,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636025 2024-12-10T11:03:56,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636025 2024-12-10T11:03:56,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636025) roll requested 2024-12-10T11:03:56,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636031 2024-12-10T11:03:56,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636031 2024-12-10T11:03:56,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636031) roll requested 2024-12-10T11:03:56,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636037 2024-12-10T11:03:56,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636037 2024-12-10T11:03:56,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636037) roll requested 2024-12-10T11:03:56,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636043 2024-12-10T11:03:56,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636043 2024-12-10T11:03:56,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636043) roll requested 2024-12-10T11:03:56,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636052 2024-12-10T11:03:56,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636052 2024-12-10T11:03:56,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636052) roll requested 2024-12-10T11:03:56,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636058 2024-12-10T11:03:56,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636058 2024-12-10T11:03:56,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636058) roll requested 2024-12-10T11:03:56,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636065 2024-12-10T11:03:56,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636065 2024-12-10T11:03:56,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636065) roll requested 2024-12-10T11:03:56,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636071 2024-12-10T11:03:56,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636071 2024-12-10T11:03:56,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636071) roll requested 2024-12-10T11:03:56,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636077 2024-12-10T11:03:56,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636077 2024-12-10T11:03:56,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636077) roll requested 2024-12-10T11:03:56,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636083 2024-12-10T11:03:56,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636083 2024-12-10T11:03:56,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636083) roll requested 2024-12-10T11:03:56,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636089 2024-12-10T11:03:56,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636089 2024-12-10T11:03:56,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636089) roll requested 2024-12-10T11:03:56,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636095 2024-12-10T11:03:56,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636095 2024-12-10T11:03:56,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636095) roll requested 2024-12-10T11:03:56,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636101 2024-12-10T11:03:56,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636101 2024-12-10T11:03:56,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636101) roll requested 2024-12-10T11:03:56,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636107 2024-12-10T11:03:56,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636107 2024-12-10T11:03:56,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636107) roll requested 2024-12-10T11:03:56,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636114 2024-12-10T11:03:56,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636114 2024-12-10T11:03:56,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636114) roll requested 2024-12-10T11:03:56,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636121 2024-12-10T11:03:56,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636121 2024-12-10T11:03:56,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636121) roll requested 2024-12-10T11:03:56,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636127 2024-12-10T11:03:56,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636127 2024-12-10T11:03:56,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636127) roll requested 2024-12-10T11:03:56,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636133 2024-12-10T11:03:56,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636133 2024-12-10T11:03:56,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636133) roll requested 2024-12-10T11:03:56,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636140 2024-12-10T11:03:56,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636140 2024-12-10T11:03:56,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636140) roll requested 2024-12-10T11:03:56,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636146 2024-12-10T11:03:56,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636146 2024-12-10T11:03:56,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636146) roll requested 2024-12-10T11:03:56,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636152 2024-12-10T11:03:56,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636152 2024-12-10T11:03:56,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636152) roll requested 2024-12-10T11:03:56,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636159 2024-12-10T11:03:56,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636159 2024-12-10T11:03:56,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636159) roll requested 2024-12-10T11:03:56,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636165 2024-12-10T11:03:56,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636165 2024-12-10T11:03:56,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636165) roll requested 2024-12-10T11:03:56,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636171 2024-12-10T11:03:56,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636171 2024-12-10T11:03:56,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636171) roll requested 2024-12-10T11:03:56,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636177 2024-12-10T11:03:56,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636177 2024-12-10T11:03:56,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636177) roll requested 2024-12-10T11:03:56,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636183 2024-12-10T11:03:56,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636183 2024-12-10T11:03:56,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636183) roll requested 2024-12-10T11:03:56,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636189 2024-12-10T11:03:56,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636189 2024-12-10T11:03:56,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636189) roll requested 2024-12-10T11:03:56,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636201 2024-12-10T11:03:56,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636201 2024-12-10T11:03:56,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636201) roll requested 2024-12-10T11:03:56,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636213 2024-12-10T11:03:56,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636213 2024-12-10T11:03:56,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636213) roll requested 2024-12-10T11:03:56,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636221 2024-12-10T11:03:56,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636221 2024-12-10T11:03:56,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636221) roll requested 2024-12-10T11:03:56,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636227 2024-12-10T11:03:56,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636227 2024-12-10T11:03:56,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636227) roll requested 2024-12-10T11:03:56,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636238 2024-12-10T11:03:56,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636238 2024-12-10T11:03:56,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636238) roll requested 2024-12-10T11:03:56,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636244 2024-12-10T11:03:56,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636244 2024-12-10T11:03:56,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636244) roll requested 2024-12-10T11:03:56,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636250 2024-12-10T11:03:56,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636250 2024-12-10T11:03:56,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636250) roll requested 2024-12-10T11:03:56,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636257 2024-12-10T11:03:56,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636257 2024-12-10T11:03:56,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636257) roll requested 2024-12-10T11:03:56,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636263 2024-12-10T11:03:56,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636263 2024-12-10T11:03:56,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636263) roll requested 2024-12-10T11:03:56,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636269 2024-12-10T11:03:56,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636269 2024-12-10T11:03:56,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636269) roll requested 2024-12-10T11:03:56,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636275 2024-12-10T11:03:56,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636275 2024-12-10T11:03:56,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636275) roll requested 2024-12-10T11:03:56,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636282 2024-12-10T11:03:56,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636282 2024-12-10T11:03:56,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636282) roll requested 2024-12-10T11:03:56,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636288 2024-12-10T11:03:56,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636288 2024-12-10T11:03:56,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636288) roll requested 2024-12-10T11:03:56,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636294 2024-12-10T11:03:56,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636294 2024-12-10T11:03:56,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636294) roll requested 2024-12-10T11:03:56,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636301 2024-12-10T11:03:56,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636301 2024-12-10T11:03:56,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636301) roll requested 2024-12-10T11:03:56,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636307 2024-12-10T11:03:56,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636307 2024-12-10T11:03:56,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636307) roll requested 2024-12-10T11:03:56,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636313 2024-12-10T11:03:56,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636313 2024-12-10T11:03:56,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636313) roll requested 2024-12-10T11:03:56,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636319 2024-12-10T11:03:56,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636319 2024-12-10T11:03:56,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636319) roll requested 2024-12-10T11:03:56,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636326 2024-12-10T11:03:56,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636326 2024-12-10T11:03:56,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636326) roll requested 2024-12-10T11:03:56,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636333 2024-12-10T11:03:56,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636333 2024-12-10T11:03:56,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636333) roll requested 2024-12-10T11:03:56,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636343 2024-12-10T11:03:56,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636343 2024-12-10T11:03:56,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636343) roll requested 2024-12-10T11:03:56,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636349 2024-12-10T11:03:56,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636349 2024-12-10T11:03:56,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636349) roll requested 2024-12-10T11:03:56,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636355 2024-12-10T11:03:56,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636355 2024-12-10T11:03:56,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636355) roll requested 2024-12-10T11:03:56,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636365 2024-12-10T11:03:56,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636365 2024-12-10T11:03:56,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636365) roll requested 2024-12-10T11:03:56,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636372 2024-12-10T11:03:56,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636372 2024-12-10T11:03:56,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636372) roll requested 2024-12-10T11:03:56,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636378 2024-12-10T11:03:56,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636378 2024-12-10T11:03:56,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636378) roll requested 2024-12-10T11:03:56,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636388 2024-12-10T11:03:56,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636388 2024-12-10T11:03:56,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636388) roll requested 2024-12-10T11:03:56,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636397 2024-12-10T11:03:56,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636397 2024-12-10T11:03:56,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636397) roll requested 2024-12-10T11:03:56,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636403 2024-12-10T11:03:56,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636403 2024-12-10T11:03:56,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636403) roll requested 2024-12-10T11:03:56,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636411 2024-12-10T11:03:56,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636411 2024-12-10T11:03:56,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636411) roll requested 2024-12-10T11:03:56,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636417 2024-12-10T11:03:56,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636417 2024-12-10T11:03:56,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636417) roll requested 2024-12-10T11:03:56,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636423 2024-12-10T11:03:56,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636423 2024-12-10T11:03:56,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636423) roll requested 2024-12-10T11:03:56,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636429 2024-12-10T11:03:56,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636429 2024-12-10T11:03:56,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636429) roll requested 2024-12-10T11:03:56,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636435 2024-12-10T11:03:56,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636435 2024-12-10T11:03:56,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636435) roll requested 2024-12-10T11:03:56,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636442 2024-12-10T11:03:56,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636442 2024-12-10T11:03:56,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636442) roll requested 2024-12-10T11:03:56,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636448 2024-12-10T11:03:56,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636448 2024-12-10T11:03:56,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636448) roll requested 2024-12-10T11:03:56,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636454 2024-12-10T11:03:56,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636454 2024-12-10T11:03:56,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636454) roll requested 2024-12-10T11:03:56,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636459 2024-12-10T11:03:56,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636459 2024-12-10T11:03:56,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636459) roll requested 2024-12-10T11:03:56,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636466 2024-12-10T11:03:56,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636466 2024-12-10T11:03:56,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636466) roll requested 2024-12-10T11:03:56,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636472 2024-12-10T11:03:56,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636472 2024-12-10T11:03:56,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636472) roll requested 2024-12-10T11:03:56,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636478 2024-12-10T11:03:56,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636478 2024-12-10T11:03:56,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636478) roll requested 2024-12-10T11:03:56,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636484 2024-12-10T11:03:56,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636484 2024-12-10T11:03:56,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636484) roll requested 2024-12-10T11:03:56,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636490 2024-12-10T11:03:56,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636490 2024-12-10T11:03:56,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636490) roll requested 2024-12-10T11:03:56,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636496 2024-12-10T11:03:56,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636496 2024-12-10T11:03:56,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636496) roll requested 2024-12-10T11:03:56,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636503 2024-12-10T11:03:56,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636503 2024-12-10T11:03:56,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636503) roll requested 2024-12-10T11:03:56,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636509 2024-12-10T11:03:56,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636509 2024-12-10T11:03:56,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636509) roll requested 2024-12-10T11:03:56,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636515 2024-12-10T11:03:56,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636515 2024-12-10T11:03:56,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636515) roll requested 2024-12-10T11:03:56,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636521 2024-12-10T11:03:56,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636521 2024-12-10T11:03:56,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636521) roll requested 2024-12-10T11:03:56,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636527 2024-12-10T11:03:56,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636527 2024-12-10T11:03:56,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636527) roll requested 2024-12-10T11:03:56,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636533 2024-12-10T11:03:56,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636533 2024-12-10T11:03:56,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636533) roll requested 2024-12-10T11:03:56,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636539 2024-12-10T11:03:56,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636539 2024-12-10T11:03:56,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636539) roll requested 2024-12-10T11:03:56,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636545 2024-12-10T11:03:56,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636545 2024-12-10T11:03:56,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636545) roll requested 2024-12-10T11:03:56,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636551 2024-12-10T11:03:56,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636551 2024-12-10T11:03:56,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636551) roll requested 2024-12-10T11:03:56,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636557 2024-12-10T11:03:56,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636557 2024-12-10T11:03:56,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636557) roll requested 2024-12-10T11:03:56,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636563 2024-12-10T11:03:56,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636563 2024-12-10T11:03:56,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636563) roll requested 2024-12-10T11:03:56,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636570 2024-12-10T11:03:56,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636570 2024-12-10T11:03:56,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636570) roll requested 2024-12-10T11:03:56,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636579 2024-12-10T11:03:56,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636579 2024-12-10T11:03:56,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636579) roll requested 2024-12-10T11:03:56,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636586 2024-12-10T11:03:56,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636586 2024-12-10T11:03:56,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636586) roll requested 2024-12-10T11:03:56,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636592 2024-12-10T11:03:56,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636592 2024-12-10T11:03:56,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636592) roll requested 2024-12-10T11:03:56,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636599 2024-12-10T11:03:56,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636599 2024-12-10T11:03:56,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636599) roll requested 2024-12-10T11:03:56,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636605 2024-12-10T11:03:56,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636605 2024-12-10T11:03:56,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636605) roll requested 2024-12-10T11:03:56,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636612 2024-12-10T11:03:56,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636612 2024-12-10T11:03:56,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636612) roll requested 2024-12-10T11:03:56,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636619 2024-12-10T11:03:56,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636619 2024-12-10T11:03:56,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636619) roll requested 2024-12-10T11:03:56,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636626 2024-12-10T11:03:56,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636626 2024-12-10T11:03:56,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636626) roll requested 2024-12-10T11:03:56,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636632 2024-12-10T11:03:56,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636632 2024-12-10T11:03:56,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636632) roll requested 2024-12-10T11:03:56,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636639 2024-12-10T11:03:56,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636639 2024-12-10T11:03:56,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636639) roll requested 2024-12-10T11:03:56,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636645 2024-12-10T11:03:56,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636645 2024-12-10T11:03:56,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636645) roll requested 2024-12-10T11:03:56,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636651 2024-12-10T11:03:56,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636651 2024-12-10T11:03:56,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636651) roll requested 2024-12-10T11:03:56,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636658 2024-12-10T11:03:56,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636658 2024-12-10T11:03:56,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636658) roll requested 2024-12-10T11:03:56,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636665 2024-12-10T11:03:56,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636665 2024-12-10T11:03:56,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636665) roll requested 2024-12-10T11:03:56,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636673 2024-12-10T11:03:56,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636673 2024-12-10T11:03:56,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636673) roll requested 2024-12-10T11:03:56,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636680 2024-12-10T11:03:56,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636680 2024-12-10T11:03:56,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636680) roll requested 2024-12-10T11:03:56,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636687 2024-12-10T11:03:56,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636687 2024-12-10T11:03:56,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636687) roll requested 2024-12-10T11:03:56,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636693 2024-12-10T11:03:56,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636693 2024-12-10T11:03:56,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636693) roll requested 2024-12-10T11:03:56,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636700 2024-12-10T11:03:56,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636700 2024-12-10T11:03:56,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636700) roll requested 2024-12-10T11:03:56,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636706 2024-12-10T11:03:56,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636706 2024-12-10T11:03:56,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636706) roll requested 2024-12-10T11:03:56,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636713 2024-12-10T11:03:56,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636713 2024-12-10T11:03:56,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636713) roll requested 2024-12-10T11:03:56,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636719 2024-12-10T11:03:56,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636719 2024-12-10T11:03:56,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636719) roll requested 2024-12-10T11:03:56,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636725 2024-12-10T11:03:56,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636725 2024-12-10T11:03:56,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636725) roll requested 2024-12-10T11:03:56,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636731 2024-12-10T11:03:56,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636731 2024-12-10T11:03:56,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636731) roll requested 2024-12-10T11:03:56,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636738 2024-12-10T11:03:56,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636738 2024-12-10T11:03:56,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636738) roll requested 2024-12-10T11:03:56,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636744 2024-12-10T11:03:56,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636744 2024-12-10T11:03:56,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636744) roll requested 2024-12-10T11:03:56,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636750 2024-12-10T11:03:56,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636750 2024-12-10T11:03:56,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636750) roll requested 2024-12-10T11:03:56,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636756 2024-12-10T11:03:56,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636756 2024-12-10T11:03:56,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636756) roll requested 2024-12-10T11:03:56,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636765 2024-12-10T11:03:56,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636765 2024-12-10T11:03:56,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636765) roll requested 2024-12-10T11:03:56,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636771 2024-12-10T11:03:56,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636771 2024-12-10T11:03:56,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636771) roll requested 2024-12-10T11:03:56,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636777 2024-12-10T11:03:56,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636777 2024-12-10T11:03:56,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636777) roll requested 2024-12-10T11:03:56,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636784 2024-12-10T11:03:56,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636784 2024-12-10T11:03:56,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636784) roll requested 2024-12-10T11:03:56,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636790 2024-12-10T11:03:56,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636790 2024-12-10T11:03:56,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636790) roll requested 2024-12-10T11:03:56,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636796 2024-12-10T11:03:56,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636796 2024-12-10T11:03:56,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636796) roll requested 2024-12-10T11:03:56,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636803 2024-12-10T11:03:56,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636803 2024-12-10T11:03:56,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636803) roll requested 2024-12-10T11:03:56,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636809 2024-12-10T11:03:56,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636809 2024-12-10T11:03:56,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636809) roll requested 2024-12-10T11:03:56,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636815 2024-12-10T11:03:56,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636815 2024-12-10T11:03:56,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636815) roll requested 2024-12-10T11:03:56,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636821 2024-12-10T11:03:56,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636821 2024-12-10T11:03:56,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636821) roll requested 2024-12-10T11:03:56,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636828 2024-12-10T11:03:56,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636828 2024-12-10T11:03:56,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636828) roll requested 2024-12-10T11:03:56,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636834 2024-12-10T11:03:56,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636834 2024-12-10T11:03:56,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636834) roll requested 2024-12-10T11:03:56,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636844 2024-12-10T11:03:56,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636844 2024-12-10T11:03:56,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636844) roll requested 2024-12-10T11:03:56,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636851 2024-12-10T11:03:56,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636851 2024-12-10T11:03:56,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636851) roll requested 2024-12-10T11:03:56,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636856 2024-12-10T11:03:56,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636856 2024-12-10T11:03:56,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636856) roll requested 2024-12-10T11:03:56,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636862 2024-12-10T11:03:56,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636862 2024-12-10T11:03:56,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636862) roll requested 2024-12-10T11:03:56,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636869 2024-12-10T11:03:56,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636869 2024-12-10T11:03:56,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636869) roll requested 2024-12-10T11:03:56,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636875 2024-12-10T11:03:56,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636875 2024-12-10T11:03:56,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636875) roll requested 2024-12-10T11:03:56,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636881 2024-12-10T11:03:56,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636881 2024-12-10T11:03:56,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636881) roll requested 2024-12-10T11:03:56,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636887 2024-12-10T11:03:56,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636887 2024-12-10T11:03:56,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636887) roll requested 2024-12-10T11:03:56,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636893 2024-12-10T11:03:56,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636893 2024-12-10T11:03:56,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636893) roll requested 2024-12-10T11:03:56,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636900 2024-12-10T11:03:56,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636900 2024-12-10T11:03:56,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636900) roll requested 2024-12-10T11:03:56,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636906 2024-12-10T11:03:56,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636906 2024-12-10T11:03:56,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636906) roll requested 2024-12-10T11:03:56,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636912 2024-12-10T11:03:56,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636912 2024-12-10T11:03:56,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636912) roll requested 2024-12-10T11:03:56,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636918 2024-12-10T11:03:56,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636918 2024-12-10T11:03:56,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636918) roll requested 2024-12-10T11:03:56,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636924 2024-12-10T11:03:56,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636924 2024-12-10T11:03:56,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636924) roll requested 2024-12-10T11:03:56,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636930 2024-12-10T11:03:56,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636930 2024-12-10T11:03:56,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636930) roll requested 2024-12-10T11:03:56,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636936 2024-12-10T11:03:56,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636936 2024-12-10T11:03:56,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636936) roll requested 2024-12-10T11:03:56,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636942 2024-12-10T11:03:56,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636942 2024-12-10T11:03:56,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636942) roll requested 2024-12-10T11:03:56,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636948 2024-12-10T11:03:56,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636948 2024-12-10T11:03:56,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636948) roll requested 2024-12-10T11:03:56,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636954 2024-12-10T11:03:56,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636954 2024-12-10T11:03:56,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636954) roll requested 2024-12-10T11:03:56,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636960 2024-12-10T11:03:56,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636960 2024-12-10T11:03:56,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636960) roll requested 2024-12-10T11:03:56,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636966 2024-12-10T11:03:56,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636966 2024-12-10T11:03:56,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636966) roll requested 2024-12-10T11:03:56,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636972 2024-12-10T11:03:56,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636972 2024-12-10T11:03:56,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636972) roll requested 2024-12-10T11:03:56,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636978 2024-12-10T11:03:56,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636978 2024-12-10T11:03:56,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636978) roll requested 2024-12-10T11:03:56,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636984 2024-12-10T11:03:56,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636984 2024-12-10T11:03:56,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636984) roll requested 2024-12-10T11:03:56,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636991 2024-12-10T11:03:56,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:56,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636991 2024-12-10T11:03:56,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:56,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:56,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636991) roll requested 2024-12-10T11:03:56,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828636997 2024-12-10T11:03:57,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636997 2024-12-10T11:03:57,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828636997) roll requested 2024-12-10T11:03:57,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637004 2024-12-10T11:03:57,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828636997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637004 2024-12-10T11:03:57,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637004) roll requested 2024-12-10T11:03:57,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637010 2024-12-10T11:03:57,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637010 2024-12-10T11:03:57,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637010) roll requested 2024-12-10T11:03:57,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637016 2024-12-10T11:03:57,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637016 2024-12-10T11:03:57,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637016) roll requested 2024-12-10T11:03:57,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637022 2024-12-10T11:03:57,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637022 2024-12-10T11:03:57,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637022) roll requested 2024-12-10T11:03:57,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637028 2024-12-10T11:03:57,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637028 2024-12-10T11:03:57,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637028) roll requested 2024-12-10T11:03:57,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637034 2024-12-10T11:03:57,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637034 2024-12-10T11:03:57,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637034) roll requested 2024-12-10T11:03:57,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637040 2024-12-10T11:03:57,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637040 2024-12-10T11:03:57,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637040) roll requested 2024-12-10T11:03:57,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637046 2024-12-10T11:03:57,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637046 2024-12-10T11:03:57,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637046) roll requested 2024-12-10T11:03:57,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637052 2024-12-10T11:03:57,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637052 2024-12-10T11:03:57,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637052) roll requested 2024-12-10T11:03:57,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637058 2024-12-10T11:03:57,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637058 2024-12-10T11:03:57,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637058) roll requested 2024-12-10T11:03:57,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637064 2024-12-10T11:03:57,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637064 2024-12-10T11:03:57,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637064) roll requested 2024-12-10T11:03:57,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637071 2024-12-10T11:03:57,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637071 2024-12-10T11:03:57,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637071) roll requested 2024-12-10T11:03:57,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637077 2024-12-10T11:03:57,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637077 2024-12-10T11:03:57,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637077) roll requested 2024-12-10T11:03:57,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637083 2024-12-10T11:03:57,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637083 2024-12-10T11:03:57,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637083) roll requested 2024-12-10T11:03:57,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637089 2024-12-10T11:03:57,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637089 2024-12-10T11:03:57,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637089) roll requested 2024-12-10T11:03:57,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637096 2024-12-10T11:03:57,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637096 2024-12-10T11:03:57,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637096) roll requested 2024-12-10T11:03:57,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637102 2024-12-10T11:03:57,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637102 2024-12-10T11:03:57,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637102) roll requested 2024-12-10T11:03:57,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637108 2024-12-10T11:03:57,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637108 2024-12-10T11:03:57,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637108) roll requested 2024-12-10T11:03:57,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637114 2024-12-10T11:03:57,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637114 2024-12-10T11:03:57,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637114) roll requested 2024-12-10T11:03:57,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637120 2024-12-10T11:03:57,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637120 2024-12-10T11:03:57,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637120) roll requested 2024-12-10T11:03:57,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637127 2024-12-10T11:03:57,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637127 2024-12-10T11:03:57,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637127) roll requested 2024-12-10T11:03:57,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637133 2024-12-10T11:03:57,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637133 2024-12-10T11:03:57,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637133) roll requested 2024-12-10T11:03:57,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637140 2024-12-10T11:03:57,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637140 2024-12-10T11:03:57,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637140) roll requested 2024-12-10T11:03:57,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637146 2024-12-10T11:03:57,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637146 2024-12-10T11:03:57,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637146) roll requested 2024-12-10T11:03:57,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637153 2024-12-10T11:03:57,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637153 2024-12-10T11:03:57,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637153) roll requested 2024-12-10T11:03:57,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637159 2024-12-10T11:03:57,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637159 2024-12-10T11:03:57,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637159) roll requested 2024-12-10T11:03:57,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637165 2024-12-10T11:03:57,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637165 2024-12-10T11:03:57,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637165) roll requested 2024-12-10T11:03:57,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637171 2024-12-10T11:03:57,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637171 2024-12-10T11:03:57,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637171) roll requested 2024-12-10T11:03:57,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637177 2024-12-10T11:03:57,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637177 2024-12-10T11:03:57,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637177) roll requested 2024-12-10T11:03:57,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637184 2024-12-10T11:03:57,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637184 2024-12-10T11:03:57,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637184) roll requested 2024-12-10T11:03:57,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637190 2024-12-10T11:03:57,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637190 2024-12-10T11:03:57,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637190) roll requested 2024-12-10T11:03:57,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637196 2024-12-10T11:03:57,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637196 2024-12-10T11:03:57,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637196) roll requested 2024-12-10T11:03:57,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637202 2024-12-10T11:03:57,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637202 2024-12-10T11:03:57,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637202) roll requested 2024-12-10T11:03:57,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637208 2024-12-10T11:03:57,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637208 2024-12-10T11:03:57,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637208) roll requested 2024-12-10T11:03:57,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637214 2024-12-10T11:03:57,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637214 2024-12-10T11:03:57,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637214) roll requested 2024-12-10T11:03:57,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637220 2024-12-10T11:03:57,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637220 2024-12-10T11:03:57,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637220) roll requested 2024-12-10T11:03:57,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637226 2024-12-10T11:03:57,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637226 2024-12-10T11:03:57,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637226) roll requested 2024-12-10T11:03:57,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637232 2024-12-10T11:03:57,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637232 2024-12-10T11:03:57,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637232) roll requested 2024-12-10T11:03:57,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637238 2024-12-10T11:03:57,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637238 2024-12-10T11:03:57,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637238) roll requested 2024-12-10T11:03:57,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637244 2024-12-10T11:03:57,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637244 2024-12-10T11:03:57,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637244) roll requested 2024-12-10T11:03:57,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637250 2024-12-10T11:03:57,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637250 2024-12-10T11:03:57,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637250) roll requested 2024-12-10T11:03:57,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637256 2024-12-10T11:03:57,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637256 2024-12-10T11:03:57,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637256) roll requested 2024-12-10T11:03:57,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637262 2024-12-10T11:03:57,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637262 2024-12-10T11:03:57,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637262) roll requested 2024-12-10T11:03:57,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637268 2024-12-10T11:03:57,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637268 2024-12-10T11:03:57,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637268) roll requested 2024-12-10T11:03:57,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637274 2024-12-10T11:03:57,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637274 2024-12-10T11:03:57,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637274) roll requested 2024-12-10T11:03:57,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637280 2024-12-10T11:03:57,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637280 2024-12-10T11:03:57,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637280) roll requested 2024-12-10T11:03:57,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637287 2024-12-10T11:03:57,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637287 2024-12-10T11:03:57,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637287) roll requested 2024-12-10T11:03:57,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637293 2024-12-10T11:03:57,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637293 2024-12-10T11:03:57,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637293) roll requested 2024-12-10T11:03:57,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637299 2024-12-10T11:03:57,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637299 2024-12-10T11:03:57,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637299) roll requested 2024-12-10T11:03:57,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637306 2024-12-10T11:03:57,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637306 2024-12-10T11:03:57,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637306) roll requested 2024-12-10T11:03:57,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637314 2024-12-10T11:03:57,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637314 2024-12-10T11:03:57,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637314) roll requested 2024-12-10T11:03:57,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637320 2024-12-10T11:03:57,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637320 2024-12-10T11:03:57,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637320) roll requested 2024-12-10T11:03:57,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637326 2024-12-10T11:03:57,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637326 2024-12-10T11:03:57,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637326) roll requested 2024-12-10T11:03:57,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637337 2024-12-10T11:03:57,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637337 2024-12-10T11:03:57,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637337) roll requested 2024-12-10T11:03:57,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637344 2024-12-10T11:03:57,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637344 2024-12-10T11:03:57,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637344) roll requested 2024-12-10T11:03:57,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637350 2024-12-10T11:03:57,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637350 2024-12-10T11:03:57,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637350) roll requested 2024-12-10T11:03:57,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637356 2024-12-10T11:03:57,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637356 2024-12-10T11:03:57,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637356) roll requested 2024-12-10T11:03:57,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637362 2024-12-10T11:03:57,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637362 2024-12-10T11:03:57,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637362) roll requested 2024-12-10T11:03:57,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637368 2024-12-10T11:03:57,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637368 2024-12-10T11:03:57,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637368) roll requested 2024-12-10T11:03:57,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637374 2024-12-10T11:03:57,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637374 2024-12-10T11:03:57,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637374) roll requested 2024-12-10T11:03:57,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637380 2024-12-10T11:03:57,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637380 2024-12-10T11:03:57,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637380) roll requested 2024-12-10T11:03:57,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637386 2024-12-10T11:03:57,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637386 2024-12-10T11:03:57,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637386) roll requested 2024-12-10T11:03:57,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637392 2024-12-10T11:03:57,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637392 2024-12-10T11:03:57,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637392) roll requested 2024-12-10T11:03:57,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637398 2024-12-10T11:03:57,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637398 2024-12-10T11:03:57,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637398) roll requested 2024-12-10T11:03:57,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637404 2024-12-10T11:03:57,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637404 2024-12-10T11:03:57,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637404) roll requested 2024-12-10T11:03:57,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637410 2024-12-10T11:03:57,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637410 2024-12-10T11:03:57,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637410) roll requested 2024-12-10T11:03:57,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637416 2024-12-10T11:03:57,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637416 2024-12-10T11:03:57,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637416) roll requested 2024-12-10T11:03:57,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637422 2024-12-10T11:03:57,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637422 2024-12-10T11:03:57,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637422) roll requested 2024-12-10T11:03:57,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637429 2024-12-10T11:03:57,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637429 2024-12-10T11:03:57,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637429) roll requested 2024-12-10T11:03:57,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637435 2024-12-10T11:03:57,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637435 2024-12-10T11:03:57,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637435) roll requested 2024-12-10T11:03:57,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637441 2024-12-10T11:03:57,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637441 2024-12-10T11:03:57,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637441) roll requested 2024-12-10T11:03:57,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637447 2024-12-10T11:03:57,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637447 2024-12-10T11:03:57,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637447) roll requested 2024-12-10T11:03:57,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637453 2024-12-10T11:03:57,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637453 2024-12-10T11:03:57,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637453) roll requested 2024-12-10T11:03:57,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637459 2024-12-10T11:03:57,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637459 2024-12-10T11:03:57,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637459) roll requested 2024-12-10T11:03:57,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637465 2024-12-10T11:03:57,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637465 2024-12-10T11:03:57,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637465) roll requested 2024-12-10T11:03:57,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637471 2024-12-10T11:03:57,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637471 2024-12-10T11:03:57,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637471) roll requested 2024-12-10T11:03:57,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637477 2024-12-10T11:03:57,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637477 2024-12-10T11:03:57,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637477) roll requested 2024-12-10T11:03:57,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637483 2024-12-10T11:03:57,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637483 2024-12-10T11:03:57,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637483) roll requested 2024-12-10T11:03:57,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637490 2024-12-10T11:03:57,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637490 2024-12-10T11:03:57,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637490) roll requested 2024-12-10T11:03:57,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637496 2024-12-10T11:03:57,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637496 2024-12-10T11:03:57,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637496) roll requested 2024-12-10T11:03:57,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637502 2024-12-10T11:03:57,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637502 2024-12-10T11:03:57,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637502) roll requested 2024-12-10T11:03:57,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637509 2024-12-10T11:03:57,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637509 2024-12-10T11:03:57,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637509) roll requested 2024-12-10T11:03:57,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637515 2024-12-10T11:03:57,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637515 2024-12-10T11:03:57,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637515) roll requested 2024-12-10T11:03:57,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637521 2024-12-10T11:03:57,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637521 2024-12-10T11:03:57,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637521) roll requested 2024-12-10T11:03:57,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637527 2024-12-10T11:03:57,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637527 2024-12-10T11:03:57,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637527) roll requested 2024-12-10T11:03:57,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637534 2024-12-10T11:03:57,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637534 2024-12-10T11:03:57,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637534) roll requested 2024-12-10T11:03:57,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637540 2024-12-10T11:03:57,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637540 2024-12-10T11:03:57,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637540) roll requested 2024-12-10T11:03:57,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637546 2024-12-10T11:03:57,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637546 2024-12-10T11:03:57,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637546) roll requested 2024-12-10T11:03:57,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637552 2024-12-10T11:03:57,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637552 2024-12-10T11:03:57,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637552) roll requested 2024-12-10T11:03:57,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637558 2024-12-10T11:03:57,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637558 2024-12-10T11:03:57,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637558) roll requested 2024-12-10T11:03:57,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637564 2024-12-10T11:03:57,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637564 2024-12-10T11:03:57,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637564) roll requested 2024-12-10T11:03:57,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637570 2024-12-10T11:03:57,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637570 2024-12-10T11:03:57,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637570) roll requested 2024-12-10T11:03:57,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637577 2024-12-10T11:03:57,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637577 2024-12-10T11:03:57,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637577) roll requested 2024-12-10T11:03:57,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637583 2024-12-10T11:03:57,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637583 2024-12-10T11:03:57,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637583) roll requested 2024-12-10T11:03:57,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637589 2024-12-10T11:03:57,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637589 2024-12-10T11:03:57,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637589) roll requested 2024-12-10T11:03:57,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637595 2024-12-10T11:03:57,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637595 2024-12-10T11:03:57,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637595) roll requested 2024-12-10T11:03:57,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637601 2024-12-10T11:03:57,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637601 2024-12-10T11:03:57,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637601) roll requested 2024-12-10T11:03:57,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637607 2024-12-10T11:03:57,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637607 2024-12-10T11:03:57,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637607) roll requested 2024-12-10T11:03:57,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637614 2024-12-10T11:03:57,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637614 2024-12-10T11:03:57,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637614) roll requested 2024-12-10T11:03:57,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637620 2024-12-10T11:03:57,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637620 2024-12-10T11:03:57,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637614 is not closed yet, will try archiving it next time 2024-12-10T11:03:57,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637620) roll requested 2024-12-10T11:03:57,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637626 2024-12-10T11:03:57,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637626 2024-12-10T11:03:57,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637626) roll requested 2024-12-10T11:03:57,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637632 2024-12-10T11:03:57,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637632 2024-12-10T11:03:57,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637632) roll requested 2024-12-10T11:03:57,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637638 2024-12-10T11:03:57,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637638 2024-12-10T11:03:57,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637638) roll requested 2024-12-10T11:03:57,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637644 2024-12-10T11:03:57,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637644 2024-12-10T11:03:57,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637644) roll requested 2024-12-10T11:03:57,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637650 2024-12-10T11:03:57,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637650 2024-12-10T11:03:57,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637650) roll requested 2024-12-10T11:03:57,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637656 2024-12-10T11:03:57,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637656 2024-12-10T11:03:57,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637656) roll requested 2024-12-10T11:03:57,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637662 2024-12-10T11:03:57,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637662 2024-12-10T11:03:57,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637662) roll requested 2024-12-10T11:03:57,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637668 2024-12-10T11:03:57,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637668 2024-12-10T11:03:57,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637668) roll requested 2024-12-10T11:03:57,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637676 2024-12-10T11:03:57,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637676 2024-12-10T11:03:57,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637676) roll requested 2024-12-10T11:03:57,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637683 2024-12-10T11:03:57,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637683 2024-12-10T11:03:57,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637683) roll requested 2024-12-10T11:03:57,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637690 2024-12-10T11:03:57,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637690 2024-12-10T11:03:57,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637690) roll requested 2024-12-10T11:03:57,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637697 2024-12-10T11:03:57,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637697 2024-12-10T11:03:57,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637697) roll requested 2024-12-10T11:03:57,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637703 2024-12-10T11:03:57,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637703 2024-12-10T11:03:57,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637703) roll requested 2024-12-10T11:03:57,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637709 2024-12-10T11:03:57,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637709 2024-12-10T11:03:57,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637709) roll requested 2024-12-10T11:03:57,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637715 2024-12-10T11:03:57,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637715 2024-12-10T11:03:57,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637715) roll requested 2024-12-10T11:03:57,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637721 2024-12-10T11:03:57,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637721 2024-12-10T11:03:57,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637721) roll requested 2024-12-10T11:03:57,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637728 2024-12-10T11:03:57,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637728 2024-12-10T11:03:57,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637728) roll requested 2024-12-10T11:03:57,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637734 2024-12-10T11:03:57,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637734 2024-12-10T11:03:57,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637734) roll requested 2024-12-10T11:03:57,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637740 2024-12-10T11:03:57,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637740 2024-12-10T11:03:57,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637740) roll requested 2024-12-10T11:03:57,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637748 2024-12-10T11:03:57,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637748 2024-12-10T11:03:57,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637748) roll requested 2024-12-10T11:03:57,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637754 2024-12-10T11:03:57,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637754 2024-12-10T11:03:57,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637754) roll requested 2024-12-10T11:03:57,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637760 2024-12-10T11:03:57,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637760 2024-12-10T11:03:57,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637760) roll requested 2024-12-10T11:03:57,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637766 2024-12-10T11:03:57,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637766 2024-12-10T11:03:57,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637766) roll requested 2024-12-10T11:03:57,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637773 2024-12-10T11:03:57,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637773 2024-12-10T11:03:57,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637773) roll requested 2024-12-10T11:03:57,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637779 2024-12-10T11:03:57,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637779 2024-12-10T11:03:57,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637779) roll requested 2024-12-10T11:03:57,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637786 2024-12-10T11:03:57,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637786 2024-12-10T11:03:57,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637786) roll requested 2024-12-10T11:03:57,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637793 2024-12-10T11:03:57,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637793 2024-12-10T11:03:57,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637793) roll requested 2024-12-10T11:03:57,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637799 2024-12-10T11:03:57,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637799 2024-12-10T11:03:57,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637799) roll requested 2024-12-10T11:03:57,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637805 2024-12-10T11:03:57,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637805 2024-12-10T11:03:57,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637805) roll requested 2024-12-10T11:03:57,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637811 2024-12-10T11:03:57,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637811 2024-12-10T11:03:57,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637811) roll requested 2024-12-10T11:03:57,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637817 2024-12-10T11:03:57,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637817 2024-12-10T11:03:57,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637817) roll requested 2024-12-10T11:03:57,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637824 2024-12-10T11:03:57,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637824 2024-12-10T11:03:57,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637824) roll requested 2024-12-10T11:03:57,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637830 2024-12-10T11:03:57,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637830 2024-12-10T11:03:57,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637830) roll requested 2024-12-10T11:03:57,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637836 2024-12-10T11:03:57,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637836 2024-12-10T11:03:57,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637836) roll requested 2024-12-10T11:03:57,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637842 2024-12-10T11:03:57,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637842 2024-12-10T11:03:57,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637842) roll requested 2024-12-10T11:03:57,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637848 2024-12-10T11:03:57,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637848 2024-12-10T11:03:57,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637848) roll requested 2024-12-10T11:03:57,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637854 2024-12-10T11:03:57,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637854 2024-12-10T11:03:57,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637854) roll requested 2024-12-10T11:03:57,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637860 2024-12-10T11:03:57,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637860 2024-12-10T11:03:57,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637860) roll requested 2024-12-10T11:03:57,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637866 2024-12-10T11:03:57,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637866 2024-12-10T11:03:57,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637866) roll requested 2024-12-10T11:03:57,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637872 2024-12-10T11:03:57,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637872 2024-12-10T11:03:57,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637872) roll requested 2024-12-10T11:03:57,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637878 2024-12-10T11:03:57,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637878 2024-12-10T11:03:57,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637878) roll requested 2024-12-10T11:03:57,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637885 2024-12-10T11:03:57,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637885 2024-12-10T11:03:57,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637885) roll requested 2024-12-10T11:03:57,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637891 2024-12-10T11:03:57,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637891 2024-12-10T11:03:57,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637891) roll requested 2024-12-10T11:03:57,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637897 2024-12-10T11:03:57,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637897 2024-12-10T11:03:57,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637897) roll requested 2024-12-10T11:03:57,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637904 2024-12-10T11:03:57,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637904 2024-12-10T11:03:57,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637904) roll requested 2024-12-10T11:03:57,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637910 2024-12-10T11:03:57,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637910 2024-12-10T11:03:57,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637910) roll requested 2024-12-10T11:03:57,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637916 2024-12-10T11:03:57,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637916 2024-12-10T11:03:57,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637916) roll requested 2024-12-10T11:03:57,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637922 2024-12-10T11:03:57,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637922 2024-12-10T11:03:57,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637922) roll requested 2024-12-10T11:03:57,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637928 2024-12-10T11:03:57,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637928 2024-12-10T11:03:57,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637928) roll requested 2024-12-10T11:03:57,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637934 2024-12-10T11:03:57,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637934 2024-12-10T11:03:57,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637934) roll requested 2024-12-10T11:03:57,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637941 2024-12-10T11:03:57,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637941 2024-12-10T11:03:57,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637941) roll requested 2024-12-10T11:03:57,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637947 2024-12-10T11:03:57,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637947 2024-12-10T11:03:57,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637947) roll requested 2024-12-10T11:03:57,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637953 2024-12-10T11:03:57,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637953 2024-12-10T11:03:57,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637953) roll requested 2024-12-10T11:03:57,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637959 2024-12-10T11:03:57,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637959 2024-12-10T11:03:57,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637959) roll requested 2024-12-10T11:03:57,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637965 2024-12-10T11:03:57,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637965 2024-12-10T11:03:57,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637965) roll requested 2024-12-10T11:03:57,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637971 2024-12-10T11:03:57,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637971 2024-12-10T11:03:57,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637971) roll requested 2024-12-10T11:03:57,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637977 2024-12-10T11:03:57,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637977 2024-12-10T11:03:57,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637977) roll requested 2024-12-10T11:03:57,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637984 2024-12-10T11:03:57,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637984 2024-12-10T11:03:57,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637984) roll requested 2024-12-10T11:03:57,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637990 2024-12-10T11:03:57,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:57,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637990 2024-12-10T11:03:57,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:57,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:57,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637990) roll requested 2024-12-10T11:03:57,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828637996 2024-12-10T11:03:58,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637996 2024-12-10T11:03:58,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828637996) roll requested 2024-12-10T11:03:58,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638002 2024-12-10T11:03:58,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828637996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638002 2024-12-10T11:03:58,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638002) roll requested 2024-12-10T11:03:58,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638008 2024-12-10T11:03:58,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638008 2024-12-10T11:03:58,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638008) roll requested 2024-12-10T11:03:58,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638014 2024-12-10T11:03:58,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638014 2024-12-10T11:03:58,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638014) roll requested 2024-12-10T11:03:58,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638020 2024-12-10T11:03:58,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638020 2024-12-10T11:03:58,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638020) roll requested 2024-12-10T11:03:58,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638026 2024-12-10T11:03:58,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638026 2024-12-10T11:03:58,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638026) roll requested 2024-12-10T11:03:58,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638032 2024-12-10T11:03:58,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638032 2024-12-10T11:03:58,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638032) roll requested 2024-12-10T11:03:58,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638039 2024-12-10T11:03:58,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638039 2024-12-10T11:03:58,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638039) roll requested 2024-12-10T11:03:58,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638045 2024-12-10T11:03:58,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638045 2024-12-10T11:03:58,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638045) roll requested 2024-12-10T11:03:58,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638051 2024-12-10T11:03:58,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638051 2024-12-10T11:03:58,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638051) roll requested 2024-12-10T11:03:58,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638057 2024-12-10T11:03:58,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638057 2024-12-10T11:03:58,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638057) roll requested 2024-12-10T11:03:58,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638063 2024-12-10T11:03:58,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638063 2024-12-10T11:03:58,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638063) roll requested 2024-12-10T11:03:58,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638069 2024-12-10T11:03:58,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638069 2024-12-10T11:03:58,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638069) roll requested 2024-12-10T11:03:58,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638075 2024-12-10T11:03:58,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638075 2024-12-10T11:03:58,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638075) roll requested 2024-12-10T11:03:58,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638081 2024-12-10T11:03:58,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638081 2024-12-10T11:03:58,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638081) roll requested 2024-12-10T11:03:58,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638087 2024-12-10T11:03:58,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638087 2024-12-10T11:03:58,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638087) roll requested 2024-12-10T11:03:58,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638093 2024-12-10T11:03:58,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638093 2024-12-10T11:03:58,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638093) roll requested 2024-12-10T11:03:58,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638100 2024-12-10T11:03:58,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638100 2024-12-10T11:03:58,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638100) roll requested 2024-12-10T11:03:58,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638106 2024-12-10T11:03:58,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638106 2024-12-10T11:03:58,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638106) roll requested 2024-12-10T11:03:58,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638112 2024-12-10T11:03:58,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638112 2024-12-10T11:03:58,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638112) roll requested 2024-12-10T11:03:58,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638118 2024-12-10T11:03:58,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638118 2024-12-10T11:03:58,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638118) roll requested 2024-12-10T11:03:58,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638124 2024-12-10T11:03:58,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638124 2024-12-10T11:03:58,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638124) roll requested 2024-12-10T11:03:58,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638130 2024-12-10T11:03:58,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638130 2024-12-10T11:03:58,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638130) roll requested 2024-12-10T11:03:58,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638136 2024-12-10T11:03:58,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638136 2024-12-10T11:03:58,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638136) roll requested 2024-12-10T11:03:58,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638142 2024-12-10T11:03:58,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638142 2024-12-10T11:03:58,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638142) roll requested 2024-12-10T11:03:58,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638149 2024-12-10T11:03:58,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638149 2024-12-10T11:03:58,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638149) roll requested 2024-12-10T11:03:58,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638155 2024-12-10T11:03:58,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638155 2024-12-10T11:03:58,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638155) roll requested 2024-12-10T11:03:58,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638161 2024-12-10T11:03:58,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638161 2024-12-10T11:03:58,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638161) roll requested 2024-12-10T11:03:58,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638167 2024-12-10T11:03:58,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638167 2024-12-10T11:03:58,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638167) roll requested 2024-12-10T11:03:58,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638173 2024-12-10T11:03:58,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638173 2024-12-10T11:03:58,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638173) roll requested 2024-12-10T11:03:58,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638179 2024-12-10T11:03:58,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638179 2024-12-10T11:03:58,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638179) roll requested 2024-12-10T11:03:58,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638187 2024-12-10T11:03:58,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638187 2024-12-10T11:03:58,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638187) roll requested 2024-12-10T11:03:58,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638196 2024-12-10T11:03:58,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638196 2024-12-10T11:03:58,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638196) roll requested 2024-12-10T11:03:58,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638202 2024-12-10T11:03:58,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638202 2024-12-10T11:03:58,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638202) roll requested 2024-12-10T11:03:58,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638208 2024-12-10T11:03:58,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638208 2024-12-10T11:03:58,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638208) roll requested 2024-12-10T11:03:58,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638214 2024-12-10T11:03:58,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638214 2024-12-10T11:03:58,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638214) roll requested 2024-12-10T11:03:58,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638220 2024-12-10T11:03:58,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638220 2024-12-10T11:03:58,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638220) roll requested 2024-12-10T11:03:58,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638227 2024-12-10T11:03:58,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638227 2024-12-10T11:03:58,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638227) roll requested 2024-12-10T11:03:58,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638234 2024-12-10T11:03:58,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638234 2024-12-10T11:03:58,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638234) roll requested 2024-12-10T11:03:58,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638240 2024-12-10T11:03:58,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638240 2024-12-10T11:03:58,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638240) roll requested 2024-12-10T11:03:58,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638246 2024-12-10T11:03:58,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638246 2024-12-10T11:03:58,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638246) roll requested 2024-12-10T11:03:58,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638252 2024-12-10T11:03:58,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638252 2024-12-10T11:03:58,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638252) roll requested 2024-12-10T11:03:58,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638258 2024-12-10T11:03:58,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638258 2024-12-10T11:03:58,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638258) roll requested 2024-12-10T11:03:58,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638265 2024-12-10T11:03:58,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638265 2024-12-10T11:03:58,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638265) roll requested 2024-12-10T11:03:58,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638270 2024-12-10T11:03:58,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638270 2024-12-10T11:03:58,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638270) roll requested 2024-12-10T11:03:58,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638276 2024-12-10T11:03:58,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638276 2024-12-10T11:03:58,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638276) roll requested 2024-12-10T11:03:58,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638282 2024-12-10T11:03:58,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638282 2024-12-10T11:03:58,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638282) roll requested 2024-12-10T11:03:58,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638288 2024-12-10T11:03:58,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638288 2024-12-10T11:03:58,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638288) roll requested 2024-12-10T11:03:58,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638294 2024-12-10T11:03:58,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638294 2024-12-10T11:03:58,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638294) roll requested 2024-12-10T11:03:58,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638300 2024-12-10T11:03:58,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638300 2024-12-10T11:03:58,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638300) roll requested 2024-12-10T11:03:58,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638308 2024-12-10T11:03:58,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638308 2024-12-10T11:03:58,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638308) roll requested 2024-12-10T11:03:58,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638314 2024-12-10T11:03:58,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638314 2024-12-10T11:03:58,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638314) roll requested 2024-12-10T11:03:58,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638320 2024-12-10T11:03:58,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638320 2024-12-10T11:03:58,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638320) roll requested 2024-12-10T11:03:58,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638327 2024-12-10T11:03:58,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638327 2024-12-10T11:03:58,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638327) roll requested 2024-12-10T11:03:58,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638333 2024-12-10T11:03:58,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638333 2024-12-10T11:03:58,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638333) roll requested 2024-12-10T11:03:58,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638339 2024-12-10T11:03:58,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638339 2024-12-10T11:03:58,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638339) roll requested 2024-12-10T11:03:58,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638345 2024-12-10T11:03:58,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638345 2024-12-10T11:03:58,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638345) roll requested 2024-12-10T11:03:58,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638351 2024-12-10T11:03:58,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638351 2024-12-10T11:03:58,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638351) roll requested 2024-12-10T11:03:58,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638357 2024-12-10T11:03:58,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638357 2024-12-10T11:03:58,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638357) roll requested 2024-12-10T11:03:58,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638364 2024-12-10T11:03:58,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638364 2024-12-10T11:03:58,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638364) roll requested 2024-12-10T11:03:58,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638370 2024-12-10T11:03:58,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638370 2024-12-10T11:03:58,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638370) roll requested 2024-12-10T11:03:58,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638376 2024-12-10T11:03:58,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638376 2024-12-10T11:03:58,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638376) roll requested 2024-12-10T11:03:58,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638382 2024-12-10T11:03:58,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638382 2024-12-10T11:03:58,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638382) roll requested 2024-12-10T11:03:58,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638388 2024-12-10T11:03:58,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638388 2024-12-10T11:03:58,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638388) roll requested 2024-12-10T11:03:58,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638394 2024-12-10T11:03:58,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638394 2024-12-10T11:03:58,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638394) roll requested 2024-12-10T11:03:58,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638400 2024-12-10T11:03:58,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638400 2024-12-10T11:03:58,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638400) roll requested 2024-12-10T11:03:58,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638406 2024-12-10T11:03:58,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638406 2024-12-10T11:03:58,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638406) roll requested 2024-12-10T11:03:58,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638413 2024-12-10T11:03:58,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638413 2024-12-10T11:03:58,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638413) roll requested 2024-12-10T11:03:58,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638419 2024-12-10T11:03:58,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638419 2024-12-10T11:03:58,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638419) roll requested 2024-12-10T11:03:58,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638425 2024-12-10T11:03:58,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638425 2024-12-10T11:03:58,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638425) roll requested 2024-12-10T11:03:58,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638431 2024-12-10T11:03:58,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638431 2024-12-10T11:03:58,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638431) roll requested 2024-12-10T11:03:58,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638437 2024-12-10T11:03:58,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638437 2024-12-10T11:03:58,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638437) roll requested 2024-12-10T11:03:58,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638443 2024-12-10T11:03:58,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638443 2024-12-10T11:03:58,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638443) roll requested 2024-12-10T11:03:58,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638449 2024-12-10T11:03:58,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638449 2024-12-10T11:03:58,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638449) roll requested 2024-12-10T11:03:58,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638455 2024-12-10T11:03:58,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638455 2024-12-10T11:03:58,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638455) roll requested 2024-12-10T11:03:58,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638462 2024-12-10T11:03:58,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638462 2024-12-10T11:03:58,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638462) roll requested 2024-12-10T11:03:58,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638468 2024-12-10T11:03:58,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638468 2024-12-10T11:03:58,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638468) roll requested 2024-12-10T11:03:58,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638474 2024-12-10T11:03:58,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638474 2024-12-10T11:03:58,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638474) roll requested 2024-12-10T11:03:58,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638480 2024-12-10T11:03:58,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638480 2024-12-10T11:03:58,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638480) roll requested 2024-12-10T11:03:58,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638487 2024-12-10T11:03:58,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638487 2024-12-10T11:03:58,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638487) roll requested 2024-12-10T11:03:58,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638496 2024-12-10T11:03:58,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638496 2024-12-10T11:03:58,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638496) roll requested 2024-12-10T11:03:58,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638509 2024-12-10T11:03:58,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638509 2024-12-10T11:03:58,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638509) roll requested 2024-12-10T11:03:58,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638516 2024-12-10T11:03:58,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638516 2024-12-10T11:03:58,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638516) roll requested 2024-12-10T11:03:58,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638525 2024-12-10T11:03:58,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638525 2024-12-10T11:03:58,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638525) roll requested 2024-12-10T11:03:58,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638533 2024-12-10T11:03:58,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638533 2024-12-10T11:03:58,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638533) roll requested 2024-12-10T11:03:58,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638539 2024-12-10T11:03:58,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638539 2024-12-10T11:03:58,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638539) roll requested 2024-12-10T11:03:58,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638547 2024-12-10T11:03:58,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638547 2024-12-10T11:03:58,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638547) roll requested 2024-12-10T11:03:58,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638553 2024-12-10T11:03:58,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638553 2024-12-10T11:03:58,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638553) roll requested 2024-12-10T11:03:58,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638561 2024-12-10T11:03:58,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638561 2024-12-10T11:03:58,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638561) roll requested 2024-12-10T11:03:58,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638569 2024-12-10T11:03:58,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638569 2024-12-10T11:03:58,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638569) roll requested 2024-12-10T11:03:58,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638582 2024-12-10T11:03:58,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638582 2024-12-10T11:03:58,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638582) roll requested 2024-12-10T11:03:58,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638592 2024-12-10T11:03:58,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638592 2024-12-10T11:03:58,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638592) roll requested 2024-12-10T11:03:58,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638602 2024-12-10T11:03:58,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638602 2024-12-10T11:03:58,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638602) roll requested 2024-12-10T11:03:58,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638609 2024-12-10T11:03:58,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638609 2024-12-10T11:03:58,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638609) roll requested 2024-12-10T11:03:58,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638623 2024-12-10T11:03:58,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638623 2024-12-10T11:03:58,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638623) roll requested 2024-12-10T11:03:58,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638633 2024-12-10T11:03:58,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638633 2024-12-10T11:03:58,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638633) roll requested 2024-12-10T11:03:58,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638640 2024-12-10T11:03:58,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638640 2024-12-10T11:03:58,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638640) roll requested 2024-12-10T11:03:58,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638649 2024-12-10T11:03:58,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638649 2024-12-10T11:03:58,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638649) roll requested 2024-12-10T11:03:58,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638657 2024-12-10T11:03:58,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638657 2024-12-10T11:03:58,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638657) roll requested 2024-12-10T11:03:58,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638664 2024-12-10T11:03:58,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638664 2024-12-10T11:03:58,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638664) roll requested 2024-12-10T11:03:58,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638670 2024-12-10T11:03:58,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638670 2024-12-10T11:03:58,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638670) roll requested 2024-12-10T11:03:58,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638677 2024-12-10T11:03:58,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638677 2024-12-10T11:03:58,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638677) roll requested 2024-12-10T11:03:58,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638684 2024-12-10T11:03:58,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638684 2024-12-10T11:03:58,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638684) roll requested 2024-12-10T11:03:58,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638694 2024-12-10T11:03:58,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638694 2024-12-10T11:03:58,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638694) roll requested 2024-12-10T11:03:58,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638700 2024-12-10T11:03:58,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638700 2024-12-10T11:03:58,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638700) roll requested 2024-12-10T11:03:58,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638707 2024-12-10T11:03:58,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638707 2024-12-10T11:03:58,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638707) roll requested 2024-12-10T11:03:58,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638714 2024-12-10T11:03:58,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638714 2024-12-10T11:03:58,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638714) roll requested 2024-12-10T11:03:58,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638720 2024-12-10T11:03:58,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638720 2024-12-10T11:03:58,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638720) roll requested 2024-12-10T11:03:58,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638727 2024-12-10T11:03:58,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638727 2024-12-10T11:03:58,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638727) roll requested 2024-12-10T11:03:58,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638733 2024-12-10T11:03:58,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638733 2024-12-10T11:03:58,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638733) roll requested 2024-12-10T11:03:58,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638739 2024-12-10T11:03:58,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638739 2024-12-10T11:03:58,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638739) roll requested 2024-12-10T11:03:58,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638745 2024-12-10T11:03:58,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638745 2024-12-10T11:03:58,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638745) roll requested 2024-12-10T11:03:58,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638751 2024-12-10T11:03:58,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638751 2024-12-10T11:03:58,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638751) roll requested 2024-12-10T11:03:58,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638758 2024-12-10T11:03:58,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638758 2024-12-10T11:03:58,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638758) roll requested 2024-12-10T11:03:58,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638764 2024-12-10T11:03:58,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638764 2024-12-10T11:03:58,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638764) roll requested 2024-12-10T11:03:58,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638770 2024-12-10T11:03:58,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638770 2024-12-10T11:03:58,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638770) roll requested 2024-12-10T11:03:58,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638776 2024-12-10T11:03:58,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638776 2024-12-10T11:03:58,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638776) roll requested 2024-12-10T11:03:58,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638783 2024-12-10T11:03:58,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638783 2024-12-10T11:03:58,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638783) roll requested 2024-12-10T11:03:58,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638789 2024-12-10T11:03:58,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638789 2024-12-10T11:03:58,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638789) roll requested 2024-12-10T11:03:58,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638795 2024-12-10T11:03:58,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638795 2024-12-10T11:03:58,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638795) roll requested 2024-12-10T11:03:58,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638804 2024-12-10T11:03:58,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638804 2024-12-10T11:03:58,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638804) roll requested 2024-12-10T11:03:58,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638810 2024-12-10T11:03:58,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638810 2024-12-10T11:03:58,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638810) roll requested 2024-12-10T11:03:58,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638816 2024-12-10T11:03:58,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638816 2024-12-10T11:03:58,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638816) roll requested 2024-12-10T11:03:58,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638828 2024-12-10T11:03:58,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638828 2024-12-10T11:03:58,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638828) roll requested 2024-12-10T11:03:58,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638834 2024-12-10T11:03:58,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638834 2024-12-10T11:03:58,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638834) roll requested 2024-12-10T11:03:58,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638842 2024-12-10T11:03:58,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638842 2024-12-10T11:03:58,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638842) roll requested 2024-12-10T11:03:58,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638848 2024-12-10T11:03:58,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638848 2024-12-10T11:03:58,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638848) roll requested 2024-12-10T11:03:58,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638854 2024-12-10T11:03:58,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638854 2024-12-10T11:03:58,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638854) roll requested 2024-12-10T11:03:58,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638860 2024-12-10T11:03:58,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638860 2024-12-10T11:03:58,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638860) roll requested 2024-12-10T11:03:58,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638866 2024-12-10T11:03:58,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638866 2024-12-10T11:03:58,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638866) roll requested 2024-12-10T11:03:58,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638872 2024-12-10T11:03:58,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638872 2024-12-10T11:03:58,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638872) roll requested 2024-12-10T11:03:58,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638878 2024-12-10T11:03:58,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638878 2024-12-10T11:03:58,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638878) roll requested 2024-12-10T11:03:58,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638884 2024-12-10T11:03:58,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638884 2024-12-10T11:03:58,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638884) roll requested 2024-12-10T11:03:58,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638890 2024-12-10T11:03:58,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638890 2024-12-10T11:03:58,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638890) roll requested 2024-12-10T11:03:58,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638898 2024-12-10T11:03:58,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638898 2024-12-10T11:03:58,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638898) roll requested 2024-12-10T11:03:58,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638913 2024-12-10T11:03:58,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638913 2024-12-10T11:03:58,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638913) roll requested 2024-12-10T11:03:58,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638919 2024-12-10T11:03:58,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638919 2024-12-10T11:03:58,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638919) roll requested 2024-12-10T11:03:58,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638925 2024-12-10T11:03:58,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638925 2024-12-10T11:03:58,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638925) roll requested 2024-12-10T11:03:58,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638931 2024-12-10T11:03:58,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638931 2024-12-10T11:03:58,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638931) roll requested 2024-12-10T11:03:58,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638937 2024-12-10T11:03:58,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638937 2024-12-10T11:03:58,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638937) roll requested 2024-12-10T11:03:58,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638943 2024-12-10T11:03:58,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638943 2024-12-10T11:03:58,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638943) roll requested 2024-12-10T11:03:58,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638951 2024-12-10T11:03:58,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638951 2024-12-10T11:03:58,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638951) roll requested 2024-12-10T11:03:58,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638957 2024-12-10T11:03:58,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638957 2024-12-10T11:03:58,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638957) roll requested 2024-12-10T11:03:58,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638963 2024-12-10T11:03:58,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638963 2024-12-10T11:03:58,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638963) roll requested 2024-12-10T11:03:58,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638969 2024-12-10T11:03:58,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638969 2024-12-10T11:03:58,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638969) roll requested 2024-12-10T11:03:58,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638975 2024-12-10T11:03:58,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638975 2024-12-10T11:03:58,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638975) roll requested 2024-12-10T11:03:58,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638981 2024-12-10T11:03:58,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638981 2024-12-10T11:03:58,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638981) roll requested 2024-12-10T11:03:58,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638988 2024-12-10T11:03:58,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638988 2024-12-10T11:03:58,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:58,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:58,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638988) roll requested 2024-12-10T11:03:58,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828638994 2024-12-10T11:03:58,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:58,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638994 2024-12-10T11:03:59,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828638994) roll requested 2024-12-10T11:03:59,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639000 2024-12-10T11:03:59,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828638994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639000 2024-12-10T11:03:59,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639000) roll requested 2024-12-10T11:03:59,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639006 2024-12-10T11:03:59,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639006 2024-12-10T11:03:59,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639006) roll requested 2024-12-10T11:03:59,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639012 2024-12-10T11:03:59,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639012 2024-12-10T11:03:59,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639012) roll requested 2024-12-10T11:03:59,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639018 2024-12-10T11:03:59,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639018 2024-12-10T11:03:59,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639018) roll requested 2024-12-10T11:03:59,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639024 2024-12-10T11:03:59,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639024 2024-12-10T11:03:59,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639024) roll requested 2024-12-10T11:03:59,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639030 2024-12-10T11:03:59,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639030 2024-12-10T11:03:59,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639030) roll requested 2024-12-10T11:03:59,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639037 2024-12-10T11:03:59,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639037 2024-12-10T11:03:59,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639037) roll requested 2024-12-10T11:03:59,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639043 2024-12-10T11:03:59,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639043 2024-12-10T11:03:59,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639043) roll requested 2024-12-10T11:03:59,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639049 2024-12-10T11:03:59,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639049 2024-12-10T11:03:59,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639049) roll requested 2024-12-10T11:03:59,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639055 2024-12-10T11:03:59,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639055 2024-12-10T11:03:59,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639055) roll requested 2024-12-10T11:03:59,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639061 2024-12-10T11:03:59,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639061 2024-12-10T11:03:59,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639061) roll requested 2024-12-10T11:03:59,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639067 2024-12-10T11:03:59,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639067 2024-12-10T11:03:59,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639067) roll requested 2024-12-10T11:03:59,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639073 2024-12-10T11:03:59,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639073 2024-12-10T11:03:59,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639073) roll requested 2024-12-10T11:03:59,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639079 2024-12-10T11:03:59,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639079 2024-12-10T11:03:59,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639079) roll requested 2024-12-10T11:03:59,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639085 2024-12-10T11:03:59,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639085 2024-12-10T11:03:59,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639085) roll requested 2024-12-10T11:03:59,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639092 2024-12-10T11:03:59,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639092 2024-12-10T11:03:59,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639092) roll requested 2024-12-10T11:03:59,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639098 2024-12-10T11:03:59,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639098 2024-12-10T11:03:59,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639098) roll requested 2024-12-10T11:03:59,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639104 2024-12-10T11:03:59,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639104 2024-12-10T11:03:59,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639104) roll requested 2024-12-10T11:03:59,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639113 2024-12-10T11:03:59,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639113 2024-12-10T11:03:59,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639113) roll requested 2024-12-10T11:03:59,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639121 2024-12-10T11:03:59,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639121 2024-12-10T11:03:59,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639121) roll requested 2024-12-10T11:03:59,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639127 2024-12-10T11:03:59,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639127 2024-12-10T11:03:59,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639127) roll requested 2024-12-10T11:03:59,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639134 2024-12-10T11:03:59,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639134 2024-12-10T11:03:59,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639134) roll requested 2024-12-10T11:03:59,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639145 2024-12-10T11:03:59,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639145 2024-12-10T11:03:59,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639145) roll requested 2024-12-10T11:03:59,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639153 2024-12-10T11:03:59,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639153 2024-12-10T11:03:59,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639153) roll requested 2024-12-10T11:03:59,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639161 2024-12-10T11:03:59,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639161 2024-12-10T11:03:59,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639161) roll requested 2024-12-10T11:03:59,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639169 2024-12-10T11:03:59,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639169 2024-12-10T11:03:59,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639169) roll requested 2024-12-10T11:03:59,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639178 2024-12-10T11:03:59,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639178 2024-12-10T11:03:59,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639178) roll requested 2024-12-10T11:03:59,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639184 2024-12-10T11:03:59,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639184 2024-12-10T11:03:59,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639184) roll requested 2024-12-10T11:03:59,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639191 2024-12-10T11:03:59,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639191 2024-12-10T11:03:59,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639191) roll requested 2024-12-10T11:03:59,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639198 2024-12-10T11:03:59,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639198 2024-12-10T11:03:59,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639198) roll requested 2024-12-10T11:03:59,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639205 2024-12-10T11:03:59,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639205 2024-12-10T11:03:59,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639205) roll requested 2024-12-10T11:03:59,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639211 2024-12-10T11:03:59,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639211 2024-12-10T11:03:59,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639211) roll requested 2024-12-10T11:03:59,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639218 2024-12-10T11:03:59,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639218 2024-12-10T11:03:59,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639218) roll requested 2024-12-10T11:03:59,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639225 2024-12-10T11:03:59,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639225 2024-12-10T11:03:59,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639225) roll requested 2024-12-10T11:03:59,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639232 2024-12-10T11:03:59,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639232 2024-12-10T11:03:59,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639232) roll requested 2024-12-10T11:03:59,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639238 2024-12-10T11:03:59,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639238 2024-12-10T11:03:59,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639238) roll requested 2024-12-10T11:03:59,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639244 2024-12-10T11:03:59,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639244 2024-12-10T11:03:59,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639244) roll requested 2024-12-10T11:03:59,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639250 2024-12-10T11:03:59,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639250 2024-12-10T11:03:59,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639250) roll requested 2024-12-10T11:03:59,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639256 2024-12-10T11:03:59,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639256 2024-12-10T11:03:59,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639256) roll requested 2024-12-10T11:03:59,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639263 2024-12-10T11:03:59,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639263 2024-12-10T11:03:59,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639263) roll requested 2024-12-10T11:03:59,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639269 2024-12-10T11:03:59,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639269 2024-12-10T11:03:59,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639269) roll requested 2024-12-10T11:03:59,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639275 2024-12-10T11:03:59,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639275 2024-12-10T11:03:59,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639275) roll requested 2024-12-10T11:03:59,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639281 2024-12-10T11:03:59,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639281 2024-12-10T11:03:59,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639281) roll requested 2024-12-10T11:03:59,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639287 2024-12-10T11:03:59,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639287 2024-12-10T11:03:59,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639287) roll requested 2024-12-10T11:03:59,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639294 2024-12-10T11:03:59,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639294 2024-12-10T11:03:59,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639294) roll requested 2024-12-10T11:03:59,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639300 2024-12-10T11:03:59,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639300 2024-12-10T11:03:59,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639300) roll requested 2024-12-10T11:03:59,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639306 2024-12-10T11:03:59,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639306 2024-12-10T11:03:59,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639306) roll requested 2024-12-10T11:03:59,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639312 2024-12-10T11:03:59,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639312 2024-12-10T11:03:59,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639312) roll requested 2024-12-10T11:03:59,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639318 2024-12-10T11:03:59,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639318 2024-12-10T11:03:59,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639318) roll requested 2024-12-10T11:03:59,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639324 2024-12-10T11:03:59,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639324 2024-12-10T11:03:59,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639324) roll requested 2024-12-10T11:03:59,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639330 2024-12-10T11:03:59,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639330 2024-12-10T11:03:59,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639330) roll requested 2024-12-10T11:03:59,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639336 2024-12-10T11:03:59,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639336 2024-12-10T11:03:59,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639336) roll requested 2024-12-10T11:03:59,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639343 2024-12-10T11:03:59,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639343 2024-12-10T11:03:59,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639343) roll requested 2024-12-10T11:03:59,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639349 2024-12-10T11:03:59,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639349 2024-12-10T11:03:59,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639349) roll requested 2024-12-10T11:03:59,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639355 2024-12-10T11:03:59,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639355 2024-12-10T11:03:59,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639355) roll requested 2024-12-10T11:03:59,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639361 2024-12-10T11:03:59,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639361 2024-12-10T11:03:59,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639361) roll requested 2024-12-10T11:03:59,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639367 2024-12-10T11:03:59,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639367 2024-12-10T11:03:59,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639367) roll requested 2024-12-10T11:03:59,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639373 2024-12-10T11:03:59,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639373 2024-12-10T11:03:59,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639373) roll requested 2024-12-10T11:03:59,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639379 2024-12-10T11:03:59,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639379 2024-12-10T11:03:59,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639379) roll requested 2024-12-10T11:03:59,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639386 2024-12-10T11:03:59,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639386 2024-12-10T11:03:59,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639386) roll requested 2024-12-10T11:03:59,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639392 2024-12-10T11:03:59,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639392 2024-12-10T11:03:59,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639392) roll requested 2024-12-10T11:03:59,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639398 2024-12-10T11:03:59,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639398 2024-12-10T11:03:59,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639398) roll requested 2024-12-10T11:03:59,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639404 2024-12-10T11:03:59,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639404 2024-12-10T11:03:59,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639404) roll requested 2024-12-10T11:03:59,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639410 2024-12-10T11:03:59,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639410 2024-12-10T11:03:59,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639410) roll requested 2024-12-10T11:03:59,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639417 2024-12-10T11:03:59,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639417 2024-12-10T11:03:59,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639417) roll requested 2024-12-10T11:03:59,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639424 2024-12-10T11:03:59,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639424 2024-12-10T11:03:59,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639424) roll requested 2024-12-10T11:03:59,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639430 2024-12-10T11:03:59,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639430 2024-12-10T11:03:59,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639430) roll requested 2024-12-10T11:03:59,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639436 2024-12-10T11:03:59,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639436 2024-12-10T11:03:59,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639436) roll requested 2024-12-10T11:03:59,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639442 2024-12-10T11:03:59,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639442 2024-12-10T11:03:59,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639442) roll requested 2024-12-10T11:03:59,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639448 2024-12-10T11:03:59,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639448 2024-12-10T11:03:59,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639448) roll requested 2024-12-10T11:03:59,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639454 2024-12-10T11:03:59,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639454 2024-12-10T11:03:59,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639454) roll requested 2024-12-10T11:03:59,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639460 2024-12-10T11:03:59,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639460 2024-12-10T11:03:59,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639460) roll requested 2024-12-10T11:03:59,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639468 2024-12-10T11:03:59,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639468 2024-12-10T11:03:59,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639468) roll requested 2024-12-10T11:03:59,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639474 2024-12-10T11:03:59,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639474 2024-12-10T11:03:59,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639474) roll requested 2024-12-10T11:03:59,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639480 2024-12-10T11:03:59,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639480 2024-12-10T11:03:59,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639480) roll requested 2024-12-10T11:03:59,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639486 2024-12-10T11:03:59,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639486 2024-12-10T11:03:59,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639486) roll requested 2024-12-10T11:03:59,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639492 2024-12-10T11:03:59,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639492 2024-12-10T11:03:59,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639492) roll requested 2024-12-10T11:03:59,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639498 2024-12-10T11:03:59,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639498 2024-12-10T11:03:59,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639498) roll requested 2024-12-10T11:03:59,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639504 2024-12-10T11:03:59,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639504 2024-12-10T11:03:59,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639504) roll requested 2024-12-10T11:03:59,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639510 2024-12-10T11:03:59,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639510 2024-12-10T11:03:59,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639510) roll requested 2024-12-10T11:03:59,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639516 2024-12-10T11:03:59,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639516 2024-12-10T11:03:59,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639516) roll requested 2024-12-10T11:03:59,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639522 2024-12-10T11:03:59,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639522 2024-12-10T11:03:59,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639522) roll requested 2024-12-10T11:03:59,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639528 2024-12-10T11:03:59,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639528 2024-12-10T11:03:59,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639528) roll requested 2024-12-10T11:03:59,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639534 2024-12-10T11:03:59,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639534 2024-12-10T11:03:59,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639534) roll requested 2024-12-10T11:03:59,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639543 2024-12-10T11:03:59,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639543 2024-12-10T11:03:59,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639543) roll requested 2024-12-10T11:03:59,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639550 2024-12-10T11:03:59,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639550 2024-12-10T11:03:59,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639550) roll requested 2024-12-10T11:03:59,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639556 2024-12-10T11:03:59,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639556 2024-12-10T11:03:59,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639556) roll requested 2024-12-10T11:03:59,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639562 2024-12-10T11:03:59,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639562 2024-12-10T11:03:59,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639562) roll requested 2024-12-10T11:03:59,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639568 2024-12-10T11:03:59,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639568 2024-12-10T11:03:59,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639568) roll requested 2024-12-10T11:03:59,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639574 2024-12-10T11:03:59,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639574 2024-12-10T11:03:59,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639574) roll requested 2024-12-10T11:03:59,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639580 2024-12-10T11:03:59,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639580 2024-12-10T11:03:59,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639580) roll requested 2024-12-10T11:03:59,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639586 2024-12-10T11:03:59,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639586 2024-12-10T11:03:59,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639586) roll requested 2024-12-10T11:03:59,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639592 2024-12-10T11:03:59,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639592 2024-12-10T11:03:59,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639592) roll requested 2024-12-10T11:03:59,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639599 2024-12-10T11:03:59,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639599 2024-12-10T11:03:59,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639599) roll requested 2024-12-10T11:03:59,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639605 2024-12-10T11:03:59,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639605 2024-12-10T11:03:59,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639605) roll requested 2024-12-10T11:03:59,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639613 2024-12-10T11:03:59,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639613 2024-12-10T11:03:59,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639613) roll requested 2024-12-10T11:03:59,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639623 2024-12-10T11:03:59,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639623 2024-12-10T11:03:59,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639623) roll requested 2024-12-10T11:03:59,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639629 2024-12-10T11:03:59,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639629 2024-12-10T11:03:59,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639629) roll requested 2024-12-10T11:03:59,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639635 2024-12-10T11:03:59,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639635 2024-12-10T11:03:59,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639635) roll requested 2024-12-10T11:03:59,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639642 2024-12-10T11:03:59,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639642 2024-12-10T11:03:59,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639642) roll requested 2024-12-10T11:03:59,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639648 2024-12-10T11:03:59,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639648 2024-12-10T11:03:59,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639648) roll requested 2024-12-10T11:03:59,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639654 2024-12-10T11:03:59,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639654 2024-12-10T11:03:59,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639654) roll requested 2024-12-10T11:03:59,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639660 2024-12-10T11:03:59,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639660 2024-12-10T11:03:59,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639660) roll requested 2024-12-10T11:03:59,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639666 2024-12-10T11:03:59,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639666 2024-12-10T11:03:59,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639666) roll requested 2024-12-10T11:03:59,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639672 2024-12-10T11:03:59,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639672 2024-12-10T11:03:59,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639672) roll requested 2024-12-10T11:03:59,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639678 2024-12-10T11:03:59,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639678 2024-12-10T11:03:59,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639678) roll requested 2024-12-10T11:03:59,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639684 2024-12-10T11:03:59,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639684 2024-12-10T11:03:59,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639684) roll requested 2024-12-10T11:03:59,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639690 2024-12-10T11:03:59,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639690 2024-12-10T11:03:59,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639690) roll requested 2024-12-10T11:03:59,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639696 2024-12-10T11:03:59,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639696 2024-12-10T11:03:59,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639696) roll requested 2024-12-10T11:03:59,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639702 2024-12-10T11:03:59,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639702 2024-12-10T11:03:59,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639702) roll requested 2024-12-10T11:03:59,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639708 2024-12-10T11:03:59,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639708 2024-12-10T11:03:59,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639708) roll requested 2024-12-10T11:03:59,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639714 2024-12-10T11:03:59,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639714 2024-12-10T11:03:59,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639714) roll requested 2024-12-10T11:03:59,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639720 2024-12-10T11:03:59,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639720 2024-12-10T11:03:59,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639720) roll requested 2024-12-10T11:03:59,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639726 2024-12-10T11:03:59,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639726 2024-12-10T11:03:59,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639726) roll requested 2024-12-10T11:03:59,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639732 2024-12-10T11:03:59,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639732 2024-12-10T11:03:59,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639732) roll requested 2024-12-10T11:03:59,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639739 2024-12-10T11:03:59,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639739 2024-12-10T11:03:59,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639739) roll requested 2024-12-10T11:03:59,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639745 2024-12-10T11:03:59,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639745 2024-12-10T11:03:59,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639745) roll requested 2024-12-10T11:03:59,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639751 2024-12-10T11:03:59,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639751 2024-12-10T11:03:59,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639751) roll requested 2024-12-10T11:03:59,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639757 2024-12-10T11:03:59,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639757 2024-12-10T11:03:59,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639757) roll requested 2024-12-10T11:03:59,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639763 2024-12-10T11:03:59,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639763 2024-12-10T11:03:59,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639763) roll requested 2024-12-10T11:03:59,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639770 2024-12-10T11:03:59,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639770 2024-12-10T11:03:59,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639770) roll requested 2024-12-10T11:03:59,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639776 2024-12-10T11:03:59,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639776 2024-12-10T11:03:59,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639776) roll requested 2024-12-10T11:03:59,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639782 2024-12-10T11:03:59,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639782 2024-12-10T11:03:59,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639782) roll requested 2024-12-10T11:03:59,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639788 2024-12-10T11:03:59,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639788 2024-12-10T11:03:59,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639788) roll requested 2024-12-10T11:03:59,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639794 2024-12-10T11:03:59,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639794 2024-12-10T11:03:59,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639794) roll requested 2024-12-10T11:03:59,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639800 2024-12-10T11:03:59,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639800 2024-12-10T11:03:59,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639800) roll requested 2024-12-10T11:03:59,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639806 2024-12-10T11:03:59,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639806 2024-12-10T11:03:59,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639806) roll requested 2024-12-10T11:03:59,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639813 2024-12-10T11:03:59,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639813 2024-12-10T11:03:59,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639813) roll requested 2024-12-10T11:03:59,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639819 2024-12-10T11:03:59,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639819 2024-12-10T11:03:59,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639819) roll requested 2024-12-10T11:03:59,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639825 2024-12-10T11:03:59,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639825 2024-12-10T11:03:59,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639825) roll requested 2024-12-10T11:03:59,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639831 2024-12-10T11:03:59,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639831 2024-12-10T11:03:59,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639831) roll requested 2024-12-10T11:03:59,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639837 2024-12-10T11:03:59,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639837 2024-12-10T11:03:59,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639837) roll requested 2024-12-10T11:03:59,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639843 2024-12-10T11:03:59,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639843 2024-12-10T11:03:59,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639843) roll requested 2024-12-10T11:03:59,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639849 2024-12-10T11:03:59,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639849 2024-12-10T11:03:59,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639849) roll requested 2024-12-10T11:03:59,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639855 2024-12-10T11:03:59,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639855 2024-12-10T11:03:59,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639855) roll requested 2024-12-10T11:03:59,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639861 2024-12-10T11:03:59,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639861 2024-12-10T11:03:59,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639861) roll requested 2024-12-10T11:03:59,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639867 2024-12-10T11:03:59,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639867 2024-12-10T11:03:59,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639867) roll requested 2024-12-10T11:03:59,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639873 2024-12-10T11:03:59,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639873 2024-12-10T11:03:59,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639873) roll requested 2024-12-10T11:03:59,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639879 2024-12-10T11:03:59,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639879 2024-12-10T11:03:59,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639879) roll requested 2024-12-10T11:03:59,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639886 2024-12-10T11:03:59,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639886 2024-12-10T11:03:59,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639886) roll requested 2024-12-10T11:03:59,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639892 2024-12-10T11:03:59,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639892 2024-12-10T11:03:59,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639892) roll requested 2024-12-10T11:03:59,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639898 2024-12-10T11:03:59,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639898 2024-12-10T11:03:59,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639898) roll requested 2024-12-10T11:03:59,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639904 2024-12-10T11:03:59,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639904 2024-12-10T11:03:59,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639904) roll requested 2024-12-10T11:03:59,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639910 2024-12-10T11:03:59,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639910 2024-12-10T11:03:59,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639910) roll requested 2024-12-10T11:03:59,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639917 2024-12-10T11:03:59,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639917 2024-12-10T11:03:59,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639917) roll requested 2024-12-10T11:03:59,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639923 2024-12-10T11:03:59,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639923 2024-12-10T11:03:59,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639923) roll requested 2024-12-10T11:03:59,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639929 2024-12-10T11:03:59,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639929 2024-12-10T11:03:59,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639929) roll requested 2024-12-10T11:03:59,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639936 2024-12-10T11:03:59,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639936 2024-12-10T11:03:59,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639936) roll requested 2024-12-10T11:03:59,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639942 2024-12-10T11:03:59,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639942 2024-12-10T11:03:59,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639942) roll requested 2024-12-10T11:03:59,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639949 2024-12-10T11:03:59,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639949 2024-12-10T11:03:59,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639949) roll requested 2024-12-10T11:03:59,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639955 2024-12-10T11:03:59,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639955 2024-12-10T11:03:59,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639955) roll requested 2024-12-10T11:03:59,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639961 2024-12-10T11:03:59,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639961 2024-12-10T11:03:59,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639961) roll requested 2024-12-10T11:03:59,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639968 2024-12-10T11:03:59,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639968 2024-12-10T11:03:59,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639968) roll requested 2024-12-10T11:03:59,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639974 2024-12-10T11:03:59,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639974 2024-12-10T11:03:59,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639974) roll requested 2024-12-10T11:03:59,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639980 2024-12-10T11:03:59,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639980 2024-12-10T11:03:59,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639980) roll requested 2024-12-10T11:03:59,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639987 2024-12-10T11:03:59,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639987 2024-12-10T11:03:59,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639987) roll requested 2024-12-10T11:03:59,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639993 2024-12-10T11:03:59,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:03:59,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639993 2024-12-10T11:03:59,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:03:59,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:03:59,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639993) roll requested 2024-12-10T11:03:59,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828639999 2024-12-10T11:04:00,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639999 2024-12-10T11:04:00,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828639999) roll requested 2024-12-10T11:04:00,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640005 2024-12-10T11:04:00,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828639999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640005 2024-12-10T11:04:00,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640005) roll requested 2024-12-10T11:04:00,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640012 2024-12-10T11:04:00,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640012 2024-12-10T11:04:00,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640012) roll requested 2024-12-10T11:04:00,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640018 2024-12-10T11:04:00,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640018 2024-12-10T11:04:00,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640018) roll requested 2024-12-10T11:04:00,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640024 2024-12-10T11:04:00,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640024 2024-12-10T11:04:00,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640024) roll requested 2024-12-10T11:04:00,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640030 2024-12-10T11:04:00,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640030 2024-12-10T11:04:00,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640030) roll requested 2024-12-10T11:04:00,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640037 2024-12-10T11:04:00,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640037 2024-12-10T11:04:00,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640037) roll requested 2024-12-10T11:04:00,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640043 2024-12-10T11:04:00,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640043 2024-12-10T11:04:00,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640043) roll requested 2024-12-10T11:04:00,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640049 2024-12-10T11:04:00,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640049 2024-12-10T11:04:00,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640049) roll requested 2024-12-10T11:04:00,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640055 2024-12-10T11:04:00,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640055 2024-12-10T11:04:00,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640055) roll requested 2024-12-10T11:04:00,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640061 2024-12-10T11:04:00,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640061 2024-12-10T11:04:00,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640061) roll requested 2024-12-10T11:04:00,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640068 2024-12-10T11:04:00,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640068 2024-12-10T11:04:00,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640068) roll requested 2024-12-10T11:04:00,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640074 2024-12-10T11:04:00,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640074 2024-12-10T11:04:00,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640074) roll requested 2024-12-10T11:04:00,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640080 2024-12-10T11:04:00,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640080 2024-12-10T11:04:00,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640080) roll requested 2024-12-10T11:04:00,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640086 2024-12-10T11:04:00,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640086 2024-12-10T11:04:00,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640086) roll requested 2024-12-10T11:04:00,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640093 2024-12-10T11:04:00,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640093 2024-12-10T11:04:00,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640093) roll requested 2024-12-10T11:04:00,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640099 2024-12-10T11:04:00,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640099 2024-12-10T11:04:00,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640099) roll requested 2024-12-10T11:04:00,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640105 2024-12-10T11:04:00,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640105 2024-12-10T11:04:00,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640105) roll requested 2024-12-10T11:04:00,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640111 2024-12-10T11:04:00,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640111 2024-12-10T11:04:00,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640111) roll requested 2024-12-10T11:04:00,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640118 2024-12-10T11:04:00,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640118 2024-12-10T11:04:00,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640118) roll requested 2024-12-10T11:04:00,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640124 2024-12-10T11:04:00,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640124 2024-12-10T11:04:00,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640124) roll requested 2024-12-10T11:04:00,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640130 2024-12-10T11:04:00,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640130 2024-12-10T11:04:00,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640130) roll requested 2024-12-10T11:04:00,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640137 2024-12-10T11:04:00,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640137 2024-12-10T11:04:00,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640137) roll requested 2024-12-10T11:04:00,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640143 2024-12-10T11:04:00,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640143 2024-12-10T11:04:00,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640143) roll requested 2024-12-10T11:04:00,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640149 2024-12-10T11:04:00,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640149 2024-12-10T11:04:00,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640149) roll requested 2024-12-10T11:04:00,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640156 2024-12-10T11:04:00,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640156 2024-12-10T11:04:00,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640156) roll requested 2024-12-10T11:04:00,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640162 2024-12-10T11:04:00,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640162 2024-12-10T11:04:00,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640162) roll requested 2024-12-10T11:04:00,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640168 2024-12-10T11:04:00,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640168 2024-12-10T11:04:00,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640168) roll requested 2024-12-10T11:04:00,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640174 2024-12-10T11:04:00,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640174 2024-12-10T11:04:00,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640174) roll requested 2024-12-10T11:04:00,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640181 2024-12-10T11:04:00,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640181 2024-12-10T11:04:00,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640181) roll requested 2024-12-10T11:04:00,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640188 2024-12-10T11:04:00,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640188 2024-12-10T11:04:00,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640188) roll requested 2024-12-10T11:04:00,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640194 2024-12-10T11:04:00,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640194 2024-12-10T11:04:00,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640194) roll requested 2024-12-10T11:04:00,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640200 2024-12-10T11:04:00,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640200 2024-12-10T11:04:00,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640200) roll requested 2024-12-10T11:04:00,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640206 2024-12-10T11:04:00,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640206 2024-12-10T11:04:00,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640206) roll requested 2024-12-10T11:04:00,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640213 2024-12-10T11:04:00,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640213 2024-12-10T11:04:00,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640213) roll requested 2024-12-10T11:04:00,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640220 2024-12-10T11:04:00,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640220 2024-12-10T11:04:00,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640220) roll requested 2024-12-10T11:04:00,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640226 2024-12-10T11:04:00,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640226 2024-12-10T11:04:00,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640226) roll requested 2024-12-10T11:04:00,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640232 2024-12-10T11:04:00,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640232 2024-12-10T11:04:00,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640232) roll requested 2024-12-10T11:04:00,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640238 2024-12-10T11:04:00,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640238 2024-12-10T11:04:00,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640238) roll requested 2024-12-10T11:04:00,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640244 2024-12-10T11:04:00,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640244 2024-12-10T11:04:00,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640244) roll requested 2024-12-10T11:04:00,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640251 2024-12-10T11:04:00,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640251 2024-12-10T11:04:00,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640251) roll requested 2024-12-10T11:04:00,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640257 2024-12-10T11:04:00,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640257 2024-12-10T11:04:00,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640257) roll requested 2024-12-10T11:04:00,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640263 2024-12-10T11:04:00,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640263 2024-12-10T11:04:00,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640263) roll requested 2024-12-10T11:04:00,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640269 2024-12-10T11:04:00,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640269 2024-12-10T11:04:00,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640269) roll requested 2024-12-10T11:04:00,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640276 2024-12-10T11:04:00,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640276 2024-12-10T11:04:00,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640276) roll requested 2024-12-10T11:04:00,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640282 2024-12-10T11:04:00,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640282 2024-12-10T11:04:00,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640282) roll requested 2024-12-10T11:04:00,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640289 2024-12-10T11:04:00,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640289 2024-12-10T11:04:00,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640289) roll requested 2024-12-10T11:04:00,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640295 2024-12-10T11:04:00,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640295 2024-12-10T11:04:00,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640295) roll requested 2024-12-10T11:04:00,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640301 2024-12-10T11:04:00,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640301 2024-12-10T11:04:00,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640301) roll requested 2024-12-10T11:04:00,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640307 2024-12-10T11:04:00,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640307 2024-12-10T11:04:00,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640307) roll requested 2024-12-10T11:04:00,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640313 2024-12-10T11:04:00,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640313 2024-12-10T11:04:00,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640313) roll requested 2024-12-10T11:04:00,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640319 2024-12-10T11:04:00,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640319 2024-12-10T11:04:00,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640319) roll requested 2024-12-10T11:04:00,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640325 2024-12-10T11:04:00,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640325 2024-12-10T11:04:00,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640325) roll requested 2024-12-10T11:04:00,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640332 2024-12-10T11:04:00,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640332 2024-12-10T11:04:00,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640332) roll requested 2024-12-10T11:04:00,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640338 2024-12-10T11:04:00,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640338 2024-12-10T11:04:00,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640338) roll requested 2024-12-10T11:04:00,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640345 2024-12-10T11:04:00,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640345 2024-12-10T11:04:00,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640345) roll requested 2024-12-10T11:04:00,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640351 2024-12-10T11:04:00,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640351 2024-12-10T11:04:00,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640351) roll requested 2024-12-10T11:04:00,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640358 2024-12-10T11:04:00,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640358 2024-12-10T11:04:00,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640358) roll requested 2024-12-10T11:04:00,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640364 2024-12-10T11:04:00,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640364 2024-12-10T11:04:00,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640364) roll requested 2024-12-10T11:04:00,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640370 2024-12-10T11:04:00,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640370 2024-12-10T11:04:00,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640370) roll requested 2024-12-10T11:04:00,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640376 2024-12-10T11:04:00,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640376 2024-12-10T11:04:00,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640376) roll requested 2024-12-10T11:04:00,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640382 2024-12-10T11:04:00,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640382 2024-12-10T11:04:00,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640382) roll requested 2024-12-10T11:04:00,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640389 2024-12-10T11:04:00,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640389 2024-12-10T11:04:00,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640389) roll requested 2024-12-10T11:04:00,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640395 2024-12-10T11:04:00,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640395 2024-12-10T11:04:00,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640395) roll requested 2024-12-10T11:04:00,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640401 2024-12-10T11:04:00,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640401 2024-12-10T11:04:00,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640401) roll requested 2024-12-10T11:04:00,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640407 2024-12-10T11:04:00,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640407 2024-12-10T11:04:00,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640407) roll requested 2024-12-10T11:04:00,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640413 2024-12-10T11:04:00,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640413 2024-12-10T11:04:00,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640413) roll requested 2024-12-10T11:04:00,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640420 2024-12-10T11:04:00,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640420 2024-12-10T11:04:00,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640420) roll requested 2024-12-10T11:04:00,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640426 2024-12-10T11:04:00,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640426 2024-12-10T11:04:00,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640426) roll requested 2024-12-10T11:04:00,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640437 2024-12-10T11:04:00,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640437 2024-12-10T11:04:00,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640437) roll requested 2024-12-10T11:04:00,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640444 2024-12-10T11:04:00,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640444 2024-12-10T11:04:00,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640444) roll requested 2024-12-10T11:04:00,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640451 2024-12-10T11:04:00,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640451 2024-12-10T11:04:00,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640451) roll requested 2024-12-10T11:04:00,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640457 2024-12-10T11:04:00,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640457 2024-12-10T11:04:00,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640457) roll requested 2024-12-10T11:04:00,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640463 2024-12-10T11:04:00,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640463 2024-12-10T11:04:00,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640463) roll requested 2024-12-10T11:04:00,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640469 2024-12-10T11:04:00,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640469 2024-12-10T11:04:00,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640469) roll requested 2024-12-10T11:04:00,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640475 2024-12-10T11:04:00,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640475 2024-12-10T11:04:00,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640475) roll requested 2024-12-10T11:04:00,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640481 2024-12-10T11:04:00,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640481 2024-12-10T11:04:00,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640481) roll requested 2024-12-10T11:04:00,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640487 2024-12-10T11:04:00,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640487 2024-12-10T11:04:00,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640487) roll requested 2024-12-10T11:04:00,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640496 2024-12-10T11:04:00,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640496 2024-12-10T11:04:00,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640496) roll requested 2024-12-10T11:04:00,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640502 2024-12-10T11:04:00,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640502 2024-12-10T11:04:00,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640502) roll requested 2024-12-10T11:04:00,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640508 2024-12-10T11:04:00,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640508 2024-12-10T11:04:00,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640508) roll requested 2024-12-10T11:04:00,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640514 2024-12-10T11:04:00,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640514 2024-12-10T11:04:00,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640514) roll requested 2024-12-10T11:04:00,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640520 2024-12-10T11:04:00,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640520 2024-12-10T11:04:00,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640520) roll requested 2024-12-10T11:04:00,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640526 2024-12-10T11:04:00,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640526 2024-12-10T11:04:00,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640526) roll requested 2024-12-10T11:04:00,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640534 2024-12-10T11:04:00,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640534 2024-12-10T11:04:00,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640534) roll requested 2024-12-10T11:04:00,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640540 2024-12-10T11:04:00,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640540 2024-12-10T11:04:00,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640540) roll requested 2024-12-10T11:04:00,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640546 2024-12-10T11:04:00,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640546 2024-12-10T11:04:00,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640546) roll requested 2024-12-10T11:04:00,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640552 2024-12-10T11:04:00,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640552 2024-12-10T11:04:00,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640552) roll requested 2024-12-10T11:04:00,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640559 2024-12-10T11:04:00,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640559 2024-12-10T11:04:00,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640559) roll requested 2024-12-10T11:04:00,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640565 2024-12-10T11:04:00,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640565 2024-12-10T11:04:00,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640565) roll requested 2024-12-10T11:04:00,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640571 2024-12-10T11:04:00,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640571 2024-12-10T11:04:00,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640571) roll requested 2024-12-10T11:04:00,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640578 2024-12-10T11:04:00,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640578 2024-12-10T11:04:00,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640578) roll requested 2024-12-10T11:04:00,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640584 2024-12-10T11:04:00,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640584 2024-12-10T11:04:00,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640584) roll requested 2024-12-10T11:04:00,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640590 2024-12-10T11:04:00,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640590 2024-12-10T11:04:00,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640590) roll requested 2024-12-10T11:04:00,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640596 2024-12-10T11:04:00,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640596 2024-12-10T11:04:00,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640596) roll requested 2024-12-10T11:04:00,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640602 2024-12-10T11:04:00,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640602 2024-12-10T11:04:00,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640602) roll requested 2024-12-10T11:04:00,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640608 2024-12-10T11:04:00,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640608 2024-12-10T11:04:00,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640608) roll requested 2024-12-10T11:04:00,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640615 2024-12-10T11:04:00,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640615 2024-12-10T11:04:00,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640615) roll requested 2024-12-10T11:04:00,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640621 2024-12-10T11:04:00,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640621 2024-12-10T11:04:00,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640621) roll requested 2024-12-10T11:04:00,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640628 2024-12-10T11:04:00,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640628 2024-12-10T11:04:00,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640628) roll requested 2024-12-10T11:04:00,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640634 2024-12-10T11:04:00,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640634 2024-12-10T11:04:00,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640634) roll requested 2024-12-10T11:04:00,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640641 2024-12-10T11:04:00,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640641 2024-12-10T11:04:00,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640641) roll requested 2024-12-10T11:04:00,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640647 2024-12-10T11:04:00,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640647 2024-12-10T11:04:00,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640647) roll requested 2024-12-10T11:04:00,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640653 2024-12-10T11:04:00,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640653 2024-12-10T11:04:00,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640653) roll requested 2024-12-10T11:04:00,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640659 2024-12-10T11:04:00,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640659 2024-12-10T11:04:00,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640659) roll requested 2024-12-10T11:04:00,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640665 2024-12-10T11:04:00,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640665 2024-12-10T11:04:00,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640665) roll requested 2024-12-10T11:04:00,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640671 2024-12-10T11:04:00,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640671 2024-12-10T11:04:00,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640671) roll requested 2024-12-10T11:04:00,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640678 2024-12-10T11:04:00,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640678 2024-12-10T11:04:00,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640678) roll requested 2024-12-10T11:04:00,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640684 2024-12-10T11:04:00,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640684 2024-12-10T11:04:00,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640684) roll requested 2024-12-10T11:04:00,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640690 2024-12-10T11:04:00,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640690 2024-12-10T11:04:00,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640690) roll requested 2024-12-10T11:04:00,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640696 2024-12-10T11:04:00,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640696 2024-12-10T11:04:00,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640696) roll requested 2024-12-10T11:04:00,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640702 2024-12-10T11:04:00,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640702 2024-12-10T11:04:00,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640702) roll requested 2024-12-10T11:04:00,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640708 2024-12-10T11:04:00,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640708 2024-12-10T11:04:00,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640708) roll requested 2024-12-10T11:04:00,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640714 2024-12-10T11:04:00,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640714 2024-12-10T11:04:00,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640714) roll requested 2024-12-10T11:04:00,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640720 2024-12-10T11:04:00,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640720 2024-12-10T11:04:00,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640720) roll requested 2024-12-10T11:04:00,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640728 2024-12-10T11:04:00,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640728 2024-12-10T11:04:00,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640728) roll requested 2024-12-10T11:04:00,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640734 2024-12-10T11:04:00,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640734 2024-12-10T11:04:00,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640734) roll requested 2024-12-10T11:04:00,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640741 2024-12-10T11:04:00,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640741 2024-12-10T11:04:00,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640741) roll requested 2024-12-10T11:04:00,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640747 2024-12-10T11:04:00,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640747 2024-12-10T11:04:00,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640747) roll requested 2024-12-10T11:04:00,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640754 2024-12-10T11:04:00,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640754 2024-12-10T11:04:00,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640754) roll requested 2024-12-10T11:04:00,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640760 2024-12-10T11:04:00,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640760 2024-12-10T11:04:00,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640760) roll requested 2024-12-10T11:04:00,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640766 2024-12-10T11:04:00,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640766 2024-12-10T11:04:00,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640766) roll requested 2024-12-10T11:04:00,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640772 2024-12-10T11:04:00,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640772 2024-12-10T11:04:00,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640772) roll requested 2024-12-10T11:04:00,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640778 2024-12-10T11:04:00,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640778 2024-12-10T11:04:00,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640778) roll requested 2024-12-10T11:04:00,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640784 2024-12-10T11:04:00,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640784 2024-12-10T11:04:00,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640784) roll requested 2024-12-10T11:04:00,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640791 2024-12-10T11:04:00,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640791 2024-12-10T11:04:00,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640791) roll requested 2024-12-10T11:04:00,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640797 2024-12-10T11:04:00,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640797 2024-12-10T11:04:00,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640797) roll requested 2024-12-10T11:04:00,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640803 2024-12-10T11:04:00,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640803 2024-12-10T11:04:00,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640803) roll requested 2024-12-10T11:04:00,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640809 2024-12-10T11:04:00,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640809 2024-12-10T11:04:00,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640809) roll requested 2024-12-10T11:04:00,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640815 2024-12-10T11:04:00,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640815 2024-12-10T11:04:00,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640815) roll requested 2024-12-10T11:04:00,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640821 2024-12-10T11:04:00,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640821 2024-12-10T11:04:00,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640821) roll requested 2024-12-10T11:04:00,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640827 2024-12-10T11:04:00,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640827 2024-12-10T11:04:00,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640827) roll requested 2024-12-10T11:04:00,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640834 2024-12-10T11:04:00,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640834 2024-12-10T11:04:00,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640834) roll requested 2024-12-10T11:04:00,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640840 2024-12-10T11:04:00,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640840 2024-12-10T11:04:00,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640840) roll requested 2024-12-10T11:04:00,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640846 2024-12-10T11:04:00,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640846 2024-12-10T11:04:00,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640846) roll requested 2024-12-10T11:04:00,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640852 2024-12-10T11:04:00,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640852 2024-12-10T11:04:00,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640852) roll requested 2024-12-10T11:04:00,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640858 2024-12-10T11:04:00,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640858 2024-12-10T11:04:00,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640858) roll requested 2024-12-10T11:04:00,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640864 2024-12-10T11:04:00,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640864 2024-12-10T11:04:00,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640864) roll requested 2024-12-10T11:04:00,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640870 2024-12-10T11:04:00,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640870 2024-12-10T11:04:00,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640870) roll requested 2024-12-10T11:04:00,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640877 2024-12-10T11:04:00,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640877 2024-12-10T11:04:00,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640877) roll requested 2024-12-10T11:04:00,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640884 2024-12-10T11:04:00,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640884 2024-12-10T11:04:00,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640884) roll requested 2024-12-10T11:04:00,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640890 2024-12-10T11:04:00,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640890 2024-12-10T11:04:00,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640890) roll requested 2024-12-10T11:04:00,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640896 2024-12-10T11:04:00,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640896 2024-12-10T11:04:00,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640896) roll requested 2024-12-10T11:04:00,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640902 2024-12-10T11:04:00,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640902 2024-12-10T11:04:00,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640902) roll requested 2024-12-10T11:04:00,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640909 2024-12-10T11:04:00,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640909 2024-12-10T11:04:00,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640909) roll requested 2024-12-10T11:04:00,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640915 2024-12-10T11:04:00,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640915 2024-12-10T11:04:00,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640915) roll requested 2024-12-10T11:04:00,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640921 2024-12-10T11:04:00,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640921 2024-12-10T11:04:00,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640921) roll requested 2024-12-10T11:04:00,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640927 2024-12-10T11:04:00,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640927 2024-12-10T11:04:00,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640927) roll requested 2024-12-10T11:04:00,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640934 2024-12-10T11:04:00,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640934 2024-12-10T11:04:00,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640934) roll requested 2024-12-10T11:04:00,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640940 2024-12-10T11:04:00,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640940 2024-12-10T11:04:00,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640940) roll requested 2024-12-10T11:04:00,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640946 2024-12-10T11:04:00,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640946 2024-12-10T11:04:00,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640946) roll requested 2024-12-10T11:04:00,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640953 2024-12-10T11:04:00,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640953 2024-12-10T11:04:00,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640953) roll requested 2024-12-10T11:04:00,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640959 2024-12-10T11:04:00,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640959 2024-12-10T11:04:00,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640959) roll requested 2024-12-10T11:04:00,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640965 2024-12-10T11:04:00,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640965 2024-12-10T11:04:00,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640965) roll requested 2024-12-10T11:04:00,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640972 2024-12-10T11:04:00,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640972 2024-12-10T11:04:00,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640972) roll requested 2024-12-10T11:04:00,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640978 2024-12-10T11:04:00,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640978 2024-12-10T11:04:00,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640978) roll requested 2024-12-10T11:04:00,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640984 2024-12-10T11:04:00,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640984 2024-12-10T11:04:00,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640984) roll requested 2024-12-10T11:04:00,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640990 2024-12-10T11:04:00,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:00,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640990 2024-12-10T11:04:00,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:00,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:00,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640990) roll requested 2024-12-10T11:04:00,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828640997 2024-12-10T11:04:01,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640997 2024-12-10T11:04:01,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828640997) roll requested 2024-12-10T11:04:01,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641005 2024-12-10T11:04:01,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828640997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641005 2024-12-10T11:04:01,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641005) roll requested 2024-12-10T11:04:01,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641011 2024-12-10T11:04:01,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641011 2024-12-10T11:04:01,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641011) roll requested 2024-12-10T11:04:01,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641018 2024-12-10T11:04:01,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641018 2024-12-10T11:04:01,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641018) roll requested 2024-12-10T11:04:01,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641024 2024-12-10T11:04:01,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641024 2024-12-10T11:04:01,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641024) roll requested 2024-12-10T11:04:01,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641030 2024-12-10T11:04:01,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641030 2024-12-10T11:04:01,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641030) roll requested 2024-12-10T11:04:01,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641036 2024-12-10T11:04:01,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641036 2024-12-10T11:04:01,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641036) roll requested 2024-12-10T11:04:01,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641043 2024-12-10T11:04:01,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641043 2024-12-10T11:04:01,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641043) roll requested 2024-12-10T11:04:01,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641053 2024-12-10T11:04:01,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641053 2024-12-10T11:04:01,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641053) roll requested 2024-12-10T11:04:01,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641062 2024-12-10T11:04:01,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641062 2024-12-10T11:04:01,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641062) roll requested 2024-12-10T11:04:01,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641068 2024-12-10T11:04:01,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641068 2024-12-10T11:04:01,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641068) roll requested 2024-12-10T11:04:01,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641074 2024-12-10T11:04:01,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641074 2024-12-10T11:04:01,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641074) roll requested 2024-12-10T11:04:01,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641081 2024-12-10T11:04:01,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641081 2024-12-10T11:04:01,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641081) roll requested 2024-12-10T11:04:01,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641087 2024-12-10T11:04:01,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641087 2024-12-10T11:04:01,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641087) roll requested 2024-12-10T11:04:01,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641094 2024-12-10T11:04:01,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641094 2024-12-10T11:04:01,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641094) roll requested 2024-12-10T11:04:01,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641101 2024-12-10T11:04:01,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641101 2024-12-10T11:04:01,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641101) roll requested 2024-12-10T11:04:01,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641107 2024-12-10T11:04:01,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641107 2024-12-10T11:04:01,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641107) roll requested 2024-12-10T11:04:01,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641114 2024-12-10T11:04:01,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641114 2024-12-10T11:04:01,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641114) roll requested 2024-12-10T11:04:01,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641121 2024-12-10T11:04:01,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641121 2024-12-10T11:04:01,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641121) roll requested 2024-12-10T11:04:01,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641127 2024-12-10T11:04:01,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641127 2024-12-10T11:04:01,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641127) roll requested 2024-12-10T11:04:01,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641134 2024-12-10T11:04:01,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641134 2024-12-10T11:04:01,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641134) roll requested 2024-12-10T11:04:01,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641140 2024-12-10T11:04:01,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641140 2024-12-10T11:04:01,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641140) roll requested 2024-12-10T11:04:01,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641146 2024-12-10T11:04:01,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641146 2024-12-10T11:04:01,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641146) roll requested 2024-12-10T11:04:01,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641152 2024-12-10T11:04:01,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641152 2024-12-10T11:04:01,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641152) roll requested 2024-12-10T11:04:01,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641159 2024-12-10T11:04:01,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641159 2024-12-10T11:04:01,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641159) roll requested 2024-12-10T11:04:01,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641165 2024-12-10T11:04:01,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641165 2024-12-10T11:04:01,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641165) roll requested 2024-12-10T11:04:01,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641171 2024-12-10T11:04:01,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641171 2024-12-10T11:04:01,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641171) roll requested 2024-12-10T11:04:01,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641178 2024-12-10T11:04:01,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641178 2024-12-10T11:04:01,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641178) roll requested 2024-12-10T11:04:01,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641184 2024-12-10T11:04:01,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641184 2024-12-10T11:04:01,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641184) roll requested 2024-12-10T11:04:01,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641191 2024-12-10T11:04:01,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641191 2024-12-10T11:04:01,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641191) roll requested 2024-12-10T11:04:01,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641197 2024-12-10T11:04:01,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641197 2024-12-10T11:04:01,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641197) roll requested 2024-12-10T11:04:01,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641204 2024-12-10T11:04:01,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641204 2024-12-10T11:04:01,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641204) roll requested 2024-12-10T11:04:01,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641210 2024-12-10T11:04:01,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641210 2024-12-10T11:04:01,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641210) roll requested 2024-12-10T11:04:01,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641216 2024-12-10T11:04:01,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641216 2024-12-10T11:04:01,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641216) roll requested 2024-12-10T11:04:01,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641222 2024-12-10T11:04:01,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641222 2024-12-10T11:04:01,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641222) roll requested 2024-12-10T11:04:01,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641228 2024-12-10T11:04:01,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641228 2024-12-10T11:04:01,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641228) roll requested 2024-12-10T11:04:01,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641234 2024-12-10T11:04:01,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641234 2024-12-10T11:04:01,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641234) roll requested 2024-12-10T11:04:01,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641241 2024-12-10T11:04:01,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641241 2024-12-10T11:04:01,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641241) roll requested 2024-12-10T11:04:01,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641247 2024-12-10T11:04:01,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641247 2024-12-10T11:04:01,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641247) roll requested 2024-12-10T11:04:01,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641253 2024-12-10T11:04:01,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641253 2024-12-10T11:04:01,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641253) roll requested 2024-12-10T11:04:01,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641259 2024-12-10T11:04:01,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641259 2024-12-10T11:04:01,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641259) roll requested 2024-12-10T11:04:01,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641265 2024-12-10T11:04:01,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641265 2024-12-10T11:04:01,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641265) roll requested 2024-12-10T11:04:01,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641272 2024-12-10T11:04:01,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641272 2024-12-10T11:04:01,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641272) roll requested 2024-12-10T11:04:01,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641278 2024-12-10T11:04:01,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641278 2024-12-10T11:04:01,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641278) roll requested 2024-12-10T11:04:01,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641284 2024-12-10T11:04:01,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641284 2024-12-10T11:04:01,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641284) roll requested 2024-12-10T11:04:01,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641291 2024-12-10T11:04:01,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641291 2024-12-10T11:04:01,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641291) roll requested 2024-12-10T11:04:01,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641297 2024-12-10T11:04:01,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641297 2024-12-10T11:04:01,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641297) roll requested 2024-12-10T11:04:01,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641303 2024-12-10T11:04:01,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641303 2024-12-10T11:04:01,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641303) roll requested 2024-12-10T11:04:01,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641309 2024-12-10T11:04:01,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641309 2024-12-10T11:04:01,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641309) roll requested 2024-12-10T11:04:01,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641315 2024-12-10T11:04:01,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641315 2024-12-10T11:04:01,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641315) roll requested 2024-12-10T11:04:01,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641322 2024-12-10T11:04:01,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641322 2024-12-10T11:04:01,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641322) roll requested 2024-12-10T11:04:01,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641328 2024-12-10T11:04:01,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641328 2024-12-10T11:04:01,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641328) roll requested 2024-12-10T11:04:01,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641335 2024-12-10T11:04:01,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641335 2024-12-10T11:04:01,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641335) roll requested 2024-12-10T11:04:01,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641341 2024-12-10T11:04:01,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641341 2024-12-10T11:04:01,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641341) roll requested 2024-12-10T11:04:01,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641347 2024-12-10T11:04:01,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641347 2024-12-10T11:04:01,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641347) roll requested 2024-12-10T11:04:01,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641353 2024-12-10T11:04:01,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641353 2024-12-10T11:04:01,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641353) roll requested 2024-12-10T11:04:01,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641359 2024-12-10T11:04:01,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641359 2024-12-10T11:04:01,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641359) roll requested 2024-12-10T11:04:01,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641365 2024-12-10T11:04:01,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641365 2024-12-10T11:04:01,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641365) roll requested 2024-12-10T11:04:01,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641372 2024-12-10T11:04:01,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641372 2024-12-10T11:04:01,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641372) roll requested 2024-12-10T11:04:01,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641378 2024-12-10T11:04:01,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641378 2024-12-10T11:04:01,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641378) roll requested 2024-12-10T11:04:01,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641384 2024-12-10T11:04:01,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641384 2024-12-10T11:04:01,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641384) roll requested 2024-12-10T11:04:01,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641390 2024-12-10T11:04:01,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641390 2024-12-10T11:04:01,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641390) roll requested 2024-12-10T11:04:01,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641397 2024-12-10T11:04:01,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641397 2024-12-10T11:04:01,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641397) roll requested 2024-12-10T11:04:01,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641409 2024-12-10T11:04:01,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641409 2024-12-10T11:04:01,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641409) roll requested 2024-12-10T11:04:01,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641416 2024-12-10T11:04:01,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641416 2024-12-10T11:04:01,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641416) roll requested 2024-12-10T11:04:01,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641422 2024-12-10T11:04:01,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641422 2024-12-10T11:04:01,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641422) roll requested 2024-12-10T11:04:01,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641428 2024-12-10T11:04:01,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641428 2024-12-10T11:04:01,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641428) roll requested 2024-12-10T11:04:01,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641434 2024-12-10T11:04:01,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641434 2024-12-10T11:04:01,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641434) roll requested 2024-12-10T11:04:01,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641440 2024-12-10T11:04:01,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641440 2024-12-10T11:04:01,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641440) roll requested 2024-12-10T11:04:01,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641447 2024-12-10T11:04:01,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641447 2024-12-10T11:04:01,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641447) roll requested 2024-12-10T11:04:01,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641453 2024-12-10T11:04:01,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641453 2024-12-10T11:04:01,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641453) roll requested 2024-12-10T11:04:01,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641459 2024-12-10T11:04:01,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641459 2024-12-10T11:04:01,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641459) roll requested 2024-12-10T11:04:01,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641465 2024-12-10T11:04:01,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641465 2024-12-10T11:04:01,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641465) roll requested 2024-12-10T11:04:01,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641471 2024-12-10T11:04:01,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641471 2024-12-10T11:04:01,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641471) roll requested 2024-12-10T11:04:01,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641478 2024-12-10T11:04:01,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641478 2024-12-10T11:04:01,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641478) roll requested 2024-12-10T11:04:01,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641484 2024-12-10T11:04:01,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641484 2024-12-10T11:04:01,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641484) roll requested 2024-12-10T11:04:01,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641490 2024-12-10T11:04:01,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641490 2024-12-10T11:04:01,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641490) roll requested 2024-12-10T11:04:01,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641497 2024-12-10T11:04:01,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641497 2024-12-10T11:04:01,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641497) roll requested 2024-12-10T11:04:01,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641503 2024-12-10T11:04:01,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641503 2024-12-10T11:04:01,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641503) roll requested 2024-12-10T11:04:01,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641509 2024-12-10T11:04:01,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641509 2024-12-10T11:04:01,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641509) roll requested 2024-12-10T11:04:01,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641515 2024-12-10T11:04:01,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641515 2024-12-10T11:04:01,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641515) roll requested 2024-12-10T11:04:01,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641521 2024-12-10T11:04:01,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641521 2024-12-10T11:04:01,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641521) roll requested 2024-12-10T11:04:01,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641527 2024-12-10T11:04:01,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641527 2024-12-10T11:04:01,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641527) roll requested 2024-12-10T11:04:01,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641533 2024-12-10T11:04:01,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641533 2024-12-10T11:04:01,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641533) roll requested 2024-12-10T11:04:01,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641540 2024-12-10T11:04:01,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641540 2024-12-10T11:04:01,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641540) roll requested 2024-12-10T11:04:01,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641546 2024-12-10T11:04:01,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641546 2024-12-10T11:04:01,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641546) roll requested 2024-12-10T11:04:01,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641552 2024-12-10T11:04:01,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641552 2024-12-10T11:04:01,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641552) roll requested 2024-12-10T11:04:01,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641559 2024-12-10T11:04:01,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641559 2024-12-10T11:04:01,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641559) roll requested 2024-12-10T11:04:01,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641565 2024-12-10T11:04:01,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641565 2024-12-10T11:04:01,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641565) roll requested 2024-12-10T11:04:01,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641571 2024-12-10T11:04:01,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641571 2024-12-10T11:04:01,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641571) roll requested 2024-12-10T11:04:01,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641578 2024-12-10T11:04:01,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641578 2024-12-10T11:04:01,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641578) roll requested 2024-12-10T11:04:01,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641584 2024-12-10T11:04:01,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641584 2024-12-10T11:04:01,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641584) roll requested 2024-12-10T11:04:01,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641594 2024-12-10T11:04:01,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641594 2024-12-10T11:04:01,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641594) roll requested 2024-12-10T11:04:01,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641604 2024-12-10T11:04:01,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641604 2024-12-10T11:04:01,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641604) roll requested 2024-12-10T11:04:01,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641612 2024-12-10T11:04:01,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641612 2024-12-10T11:04:01,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641612) roll requested 2024-12-10T11:04:01,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641619 2024-12-10T11:04:01,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641619 2024-12-10T11:04:01,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641619) roll requested 2024-12-10T11:04:01,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641625 2024-12-10T11:04:01,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641625 2024-12-10T11:04:01,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641625) roll requested 2024-12-10T11:04:01,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641631 2024-12-10T11:04:01,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641631 2024-12-10T11:04:01,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641631) roll requested 2024-12-10T11:04:01,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641638 2024-12-10T11:04:01,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641638 2024-12-10T11:04:01,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641638) roll requested 2024-12-10T11:04:01,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641645 2024-12-10T11:04:01,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641645 2024-12-10T11:04:01,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641645) roll requested 2024-12-10T11:04:01,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641651 2024-12-10T11:04:01,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641651 2024-12-10T11:04:01,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641651) roll requested 2024-12-10T11:04:01,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641658 2024-12-10T11:04:01,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641658 2024-12-10T11:04:01,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641658) roll requested 2024-12-10T11:04:01,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641664 2024-12-10T11:04:01,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641664 2024-12-10T11:04:01,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641664) roll requested 2024-12-10T11:04:01,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641670 2024-12-10T11:04:01,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641670 2024-12-10T11:04:01,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641670) roll requested 2024-12-10T11:04:01,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641678 2024-12-10T11:04:01,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641678 2024-12-10T11:04:01,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641678) roll requested 2024-12-10T11:04:01,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641684 2024-12-10T11:04:01,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641684 2024-12-10T11:04:01,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641684) roll requested 2024-12-10T11:04:01,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641691 2024-12-10T11:04:01,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641691 2024-12-10T11:04:01,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641691) roll requested 2024-12-10T11:04:01,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641697 2024-12-10T11:04:01,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641697 2024-12-10T11:04:01,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641697) roll requested 2024-12-10T11:04:01,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641703 2024-12-10T11:04:01,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641703 2024-12-10T11:04:01,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641703) roll requested 2024-12-10T11:04:01,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641709 2024-12-10T11:04:01,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641709 2024-12-10T11:04:01,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641709) roll requested 2024-12-10T11:04:01,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641715 2024-12-10T11:04:01,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641715 2024-12-10T11:04:01,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641715) roll requested 2024-12-10T11:04:01,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641721 2024-12-10T11:04:01,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641721 2024-12-10T11:04:01,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641721) roll requested 2024-12-10T11:04:01,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641727 2024-12-10T11:04:01,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641727 2024-12-10T11:04:01,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641727) roll requested 2024-12-10T11:04:01,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641733 2024-12-10T11:04:01,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641733 2024-12-10T11:04:01,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641733) roll requested 2024-12-10T11:04:01,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641743 2024-12-10T11:04:01,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641743 2024-12-10T11:04:01,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641743) roll requested 2024-12-10T11:04:01,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641749 2024-12-10T11:04:01,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641749 2024-12-10T11:04:01,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641749) roll requested 2024-12-10T11:04:01,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641756 2024-12-10T11:04:01,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641756 2024-12-10T11:04:01,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641756) roll requested 2024-12-10T11:04:01,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641763 2024-12-10T11:04:01,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641763 2024-12-10T11:04:01,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641763) roll requested 2024-12-10T11:04:01,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641769 2024-12-10T11:04:01,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641769 2024-12-10T11:04:01,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641769) roll requested 2024-12-10T11:04:01,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641775 2024-12-10T11:04:01,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641775 2024-12-10T11:04:01,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641775) roll requested 2024-12-10T11:04:01,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641782 2024-12-10T11:04:01,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641782 2024-12-10T11:04:01,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641782) roll requested 2024-12-10T11:04:01,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641791 2024-12-10T11:04:01,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641791 2024-12-10T11:04:01,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641791) roll requested 2024-12-10T11:04:01,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641798 2024-12-10T11:04:01,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641798 2024-12-10T11:04:01,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641798) roll requested 2024-12-10T11:04:01,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641804 2024-12-10T11:04:01,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641804 2024-12-10T11:04:01,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641804) roll requested 2024-12-10T11:04:01,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641810 2024-12-10T11:04:01,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641810 2024-12-10T11:04:01,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641810) roll requested 2024-12-10T11:04:01,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641817 2024-12-10T11:04:01,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641817 2024-12-10T11:04:01,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641817) roll requested 2024-12-10T11:04:01,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641823 2024-12-10T11:04:01,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641823 2024-12-10T11:04:01,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641823) roll requested 2024-12-10T11:04:01,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641833 2024-12-10T11:04:01,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641833 2024-12-10T11:04:01,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641833) roll requested 2024-12-10T11:04:01,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641842 2024-12-10T11:04:01,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641842 2024-12-10T11:04:01,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641842) roll requested 2024-12-10T11:04:01,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641850 2024-12-10T11:04:01,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641850 2024-12-10T11:04:01,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641850) roll requested 2024-12-10T11:04:01,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641858 2024-12-10T11:04:01,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641858 2024-12-10T11:04:01,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641858) roll requested 2024-12-10T11:04:01,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641864 2024-12-10T11:04:01,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641864 2024-12-10T11:04:01,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641864) roll requested 2024-12-10T11:04:01,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641871 2024-12-10T11:04:01,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641871 2024-12-10T11:04:01,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641871) roll requested 2024-12-10T11:04:01,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641878 2024-12-10T11:04:01,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641878 2024-12-10T11:04:01,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641878) roll requested 2024-12-10T11:04:01,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641886 2024-12-10T11:04:01,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641886 2024-12-10T11:04:01,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641886) roll requested 2024-12-10T11:04:01,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641893 2024-12-10T11:04:01,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641893 2024-12-10T11:04:01,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641893) roll requested 2024-12-10T11:04:01,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641902 2024-12-10T11:04:01,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641902 2024-12-10T11:04:01,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641902) roll requested 2024-12-10T11:04:01,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641908 2024-12-10T11:04:01,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641908 2024-12-10T11:04:01,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641908) roll requested 2024-12-10T11:04:01,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641914 2024-12-10T11:04:01,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641914 2024-12-10T11:04:01,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641914) roll requested 2024-12-10T11:04:01,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641920 2024-12-10T11:04:01,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641920 2024-12-10T11:04:01,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641920) roll requested 2024-12-10T11:04:01,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641926 2024-12-10T11:04:01,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641926 2024-12-10T11:04:01,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641926) roll requested 2024-12-10T11:04:01,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641932 2024-12-10T11:04:01,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641932 2024-12-10T11:04:01,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641932) roll requested 2024-12-10T11:04:01,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641939 2024-12-10T11:04:01,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641939 2024-12-10T11:04:01,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641939) roll requested 2024-12-10T11:04:01,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641947 2024-12-10T11:04:01,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641947 2024-12-10T11:04:01,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641947) roll requested 2024-12-10T11:04:01,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641954 2024-12-10T11:04:01,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641954 2024-12-10T11:04:01,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641954) roll requested 2024-12-10T11:04:01,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641961 2024-12-10T11:04:01,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641961 2024-12-10T11:04:01,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641961) roll requested 2024-12-10T11:04:01,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641967 2024-12-10T11:04:01,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641967 2024-12-10T11:04:01,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641967) roll requested 2024-12-10T11:04:01,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641974 2024-12-10T11:04:01,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641974 2024-12-10T11:04:01,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641974) roll requested 2024-12-10T11:04:01,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641980 2024-12-10T11:04:01,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641980 2024-12-10T11:04:01,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641980) roll requested 2024-12-10T11:04:01,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641986 2024-12-10T11:04:01,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641986 2024-12-10T11:04:01,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641986) roll requested 2024-12-10T11:04:01,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641992 2024-12-10T11:04:01,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:01,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641992 2024-12-10T11:04:01,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:01,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:01,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641992) roll requested 2024-12-10T11:04:01,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828641999 2024-12-10T11:04:02,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641999 2024-12-10T11:04:02,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828641999) roll requested 2024-12-10T11:04:02,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642013 2024-12-10T11:04:02,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828641999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642013 2024-12-10T11:04:02,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642013) roll requested 2024-12-10T11:04:02,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642020 2024-12-10T11:04:02,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642020 2024-12-10T11:04:02,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642020) roll requested 2024-12-10T11:04:02,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642026 2024-12-10T11:04:02,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642026 2024-12-10T11:04:02,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642026) roll requested 2024-12-10T11:04:02,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642033 2024-12-10T11:04:02,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642033 2024-12-10T11:04:02,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642033) roll requested 2024-12-10T11:04:02,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642039 2024-12-10T11:04:02,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642039 2024-12-10T11:04:02,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642039) roll requested 2024-12-10T11:04:02,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642046 2024-12-10T11:04:02,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642046 2024-12-10T11:04:02,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642046) roll requested 2024-12-10T11:04:02,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642052 2024-12-10T11:04:02,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642052 2024-12-10T11:04:02,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642052) roll requested 2024-12-10T11:04:02,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642058 2024-12-10T11:04:02,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642058 2024-12-10T11:04:02,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642058) roll requested 2024-12-10T11:04:02,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642064 2024-12-10T11:04:02,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642064 2024-12-10T11:04:02,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642064) roll requested 2024-12-10T11:04:02,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642070 2024-12-10T11:04:02,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642070 2024-12-10T11:04:02,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642070) roll requested 2024-12-10T11:04:02,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642076 2024-12-10T11:04:02,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642076 2024-12-10T11:04:02,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642076) roll requested 2024-12-10T11:04:02,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642083 2024-12-10T11:04:02,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642083 2024-12-10T11:04:02,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642083) roll requested 2024-12-10T11:04:02,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642089 2024-12-10T11:04:02,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642089 2024-12-10T11:04:02,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642089) roll requested 2024-12-10T11:04:02,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642095 2024-12-10T11:04:02,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642095 2024-12-10T11:04:02,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642095) roll requested 2024-12-10T11:04:02,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642109 2024-12-10T11:04:02,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642109 2024-12-10T11:04:02,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642109) roll requested 2024-12-10T11:04:02,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642116 2024-12-10T11:04:02,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642116 2024-12-10T11:04:02,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642116) roll requested 2024-12-10T11:04:02,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642124 2024-12-10T11:04:02,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642124 2024-12-10T11:04:02,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642124) roll requested 2024-12-10T11:04:02,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642130 2024-12-10T11:04:02,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642130 2024-12-10T11:04:02,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642130) roll requested 2024-12-10T11:04:02,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642137 2024-12-10T11:04:02,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642137 2024-12-10T11:04:02,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642137) roll requested 2024-12-10T11:04:02,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642143 2024-12-10T11:04:02,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642143 2024-12-10T11:04:02,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642143) roll requested 2024-12-10T11:04:02,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642149 2024-12-10T11:04:02,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642149 2024-12-10T11:04:02,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642149) roll requested 2024-12-10T11:04:02,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642157 2024-12-10T11:04:02,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642157 2024-12-10T11:04:02,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642157) roll requested 2024-12-10T11:04:02,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642164 2024-12-10T11:04:02,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642164 2024-12-10T11:04:02,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642164) roll requested 2024-12-10T11:04:02,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642170 2024-12-10T11:04:02,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642170 2024-12-10T11:04:02,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642170) roll requested 2024-12-10T11:04:02,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642176 2024-12-10T11:04:02,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642176 2024-12-10T11:04:02,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642176) roll requested 2024-12-10T11:04:02,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642185 2024-12-10T11:04:02,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642185 2024-12-10T11:04:02,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642185) roll requested 2024-12-10T11:04:02,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642192 2024-12-10T11:04:02,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642192 2024-12-10T11:04:02,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642192) roll requested 2024-12-10T11:04:02,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642199 2024-12-10T11:04:02,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642199 2024-12-10T11:04:02,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642199) roll requested 2024-12-10T11:04:02,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642206 2024-12-10T11:04:02,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642206 2024-12-10T11:04:02,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642206) roll requested 2024-12-10T11:04:02,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642212 2024-12-10T11:04:02,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642212 2024-12-10T11:04:02,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642212) roll requested 2024-12-10T11:04:02,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642219 2024-12-10T11:04:02,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642219 2024-12-10T11:04:02,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642219) roll requested 2024-12-10T11:04:02,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642226 2024-12-10T11:04:02,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642226 2024-12-10T11:04:02,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642226) roll requested 2024-12-10T11:04:02,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642232 2024-12-10T11:04:02,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642232 2024-12-10T11:04:02,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642232) roll requested 2024-12-10T11:04:02,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642239 2024-12-10T11:04:02,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642239 2024-12-10T11:04:02,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642239) roll requested 2024-12-10T11:04:02,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642245 2024-12-10T11:04:02,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642245 2024-12-10T11:04:02,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642245) roll requested 2024-12-10T11:04:02,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642251 2024-12-10T11:04:02,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642251 2024-12-10T11:04:02,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642251) roll requested 2024-12-10T11:04:02,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642257 2024-12-10T11:04:02,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642257 2024-12-10T11:04:02,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642257) roll requested 2024-12-10T11:04:02,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642263 2024-12-10T11:04:02,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642263 2024-12-10T11:04:02,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642263) roll requested 2024-12-10T11:04:02,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642270 2024-12-10T11:04:02,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642270 2024-12-10T11:04:02,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642270) roll requested 2024-12-10T11:04:02,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642276 2024-12-10T11:04:02,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642276 2024-12-10T11:04:02,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642276) roll requested 2024-12-10T11:04:02,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642283 2024-12-10T11:04:02,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642283 2024-12-10T11:04:02,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642283) roll requested 2024-12-10T11:04:02,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642290 2024-12-10T11:04:02,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642290 2024-12-10T11:04:02,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642290) roll requested 2024-12-10T11:04:02,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642296 2024-12-10T11:04:02,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642296 2024-12-10T11:04:02,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642296) roll requested 2024-12-10T11:04:02,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642302 2024-12-10T11:04:02,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642302 2024-12-10T11:04:02,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642302) roll requested 2024-12-10T11:04:02,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642308 2024-12-10T11:04:02,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642308 2024-12-10T11:04:02,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642308) roll requested 2024-12-10T11:04:02,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642314 2024-12-10T11:04:02,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642314 2024-12-10T11:04:02,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642314) roll requested 2024-12-10T11:04:02,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642321 2024-12-10T11:04:02,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642321 2024-12-10T11:04:02,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642321) roll requested 2024-12-10T11:04:02,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642327 2024-12-10T11:04:02,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642327 2024-12-10T11:04:02,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642327) roll requested 2024-12-10T11:04:02,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642333 2024-12-10T11:04:02,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642333 2024-12-10T11:04:02,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642333) roll requested 2024-12-10T11:04:02,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642339 2024-12-10T11:04:02,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642339 2024-12-10T11:04:02,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642339) roll requested 2024-12-10T11:04:02,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642346 2024-12-10T11:04:02,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642346 2024-12-10T11:04:02,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642346) roll requested 2024-12-10T11:04:02,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642352 2024-12-10T11:04:02,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642352 2024-12-10T11:04:02,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642352) roll requested 2024-12-10T11:04:02,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642359 2024-12-10T11:04:02,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642359 2024-12-10T11:04:02,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642359) roll requested 2024-12-10T11:04:02,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642365 2024-12-10T11:04:02,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642365 2024-12-10T11:04:02,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642365) roll requested 2024-12-10T11:04:02,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642371 2024-12-10T11:04:02,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642371 2024-12-10T11:04:02,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642371) roll requested 2024-12-10T11:04:02,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642377 2024-12-10T11:04:02,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642377 2024-12-10T11:04:02,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642377) roll requested 2024-12-10T11:04:02,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642384 2024-12-10T11:04:02,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642384 2024-12-10T11:04:02,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642384) roll requested 2024-12-10T11:04:02,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642390 2024-12-10T11:04:02,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642390 2024-12-10T11:04:02,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642390) roll requested 2024-12-10T11:04:02,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642396 2024-12-10T11:04:02,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642396 2024-12-10T11:04:02,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642396) roll requested 2024-12-10T11:04:02,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642402 2024-12-10T11:04:02,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642402 2024-12-10T11:04:02,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642402) roll requested 2024-12-10T11:04:02,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642408 2024-12-10T11:04:02,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642408 2024-12-10T11:04:02,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642408) roll requested 2024-12-10T11:04:02,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642414 2024-12-10T11:04:02,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642414 2024-12-10T11:04:02,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642414) roll requested 2024-12-10T11:04:02,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642420 2024-12-10T11:04:02,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642420 2024-12-10T11:04:02,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642420) roll requested 2024-12-10T11:04:02,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642426 2024-12-10T11:04:02,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642426 2024-12-10T11:04:02,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642426) roll requested 2024-12-10T11:04:02,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642433 2024-12-10T11:04:02,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642433 2024-12-10T11:04:02,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642433) roll requested 2024-12-10T11:04:02,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642439 2024-12-10T11:04:02,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642439 2024-12-10T11:04:02,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642439) roll requested 2024-12-10T11:04:02,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642445 2024-12-10T11:04:02,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642445 2024-12-10T11:04:02,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642445) roll requested 2024-12-10T11:04:02,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642451 2024-12-10T11:04:02,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642451 2024-12-10T11:04:02,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642451) roll requested 2024-12-10T11:04:02,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642458 2024-12-10T11:04:02,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642458 2024-12-10T11:04:02,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642458) roll requested 2024-12-10T11:04:02,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642464 2024-12-10T11:04:02,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642464 2024-12-10T11:04:02,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642464) roll requested 2024-12-10T11:04:02,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642470 2024-12-10T11:04:02,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642470 2024-12-10T11:04:02,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642470) roll requested 2024-12-10T11:04:02,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642476 2024-12-10T11:04:02,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642476 2024-12-10T11:04:02,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642476) roll requested 2024-12-10T11:04:02,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642483 2024-12-10T11:04:02,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642483 2024-12-10T11:04:02,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642483) roll requested 2024-12-10T11:04:02,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642489 2024-12-10T11:04:02,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642489 2024-12-10T11:04:02,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642489) roll requested 2024-12-10T11:04:02,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642495 2024-12-10T11:04:02,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642495 2024-12-10T11:04:02,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642495) roll requested 2024-12-10T11:04:02,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642501 2024-12-10T11:04:02,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642501 2024-12-10T11:04:02,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642501) roll requested 2024-12-10T11:04:02,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642507 2024-12-10T11:04:02,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642507 2024-12-10T11:04:02,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642507) roll requested 2024-12-10T11:04:02,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642513 2024-12-10T11:04:02,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642513 2024-12-10T11:04:02,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642513) roll requested 2024-12-10T11:04:02,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642519 2024-12-10T11:04:02,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642519 2024-12-10T11:04:02,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642519) roll requested 2024-12-10T11:04:02,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642525 2024-12-10T11:04:02,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642525 2024-12-10T11:04:02,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642525) roll requested 2024-12-10T11:04:02,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642532 2024-12-10T11:04:02,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642532 2024-12-10T11:04:02,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642532) roll requested 2024-12-10T11:04:02,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642538 2024-12-10T11:04:02,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642538 2024-12-10T11:04:02,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642538) roll requested 2024-12-10T11:04:02,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642544 2024-12-10T11:04:02,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642544 2024-12-10T11:04:02,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642544) roll requested 2024-12-10T11:04:02,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642550 2024-12-10T11:04:02,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642550 2024-12-10T11:04:02,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642550) roll requested 2024-12-10T11:04:02,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642558 2024-12-10T11:04:02,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642558 2024-12-10T11:04:02,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642558) roll requested 2024-12-10T11:04:02,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642566 2024-12-10T11:04:02,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642566 2024-12-10T11:04:02,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642566) roll requested 2024-12-10T11:04:02,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642574 2024-12-10T11:04:02,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642574 2024-12-10T11:04:02,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642574) roll requested 2024-12-10T11:04:02,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642582 2024-12-10T11:04:02,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642582 2024-12-10T11:04:02,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642582) roll requested 2024-12-10T11:04:02,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642588 2024-12-10T11:04:02,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642588 2024-12-10T11:04:02,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642588) roll requested 2024-12-10T11:04:02,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642597 2024-12-10T11:04:02,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642597 2024-12-10T11:04:02,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642597) roll requested 2024-12-10T11:04:02,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642604 2024-12-10T11:04:02,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642604 2024-12-10T11:04:02,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642604) roll requested 2024-12-10T11:04:02,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642611 2024-12-10T11:04:02,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642611 2024-12-10T11:04:02,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642611) roll requested 2024-12-10T11:04:02,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642618 2024-12-10T11:04:02,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642618 2024-12-10T11:04:02,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642618) roll requested 2024-12-10T11:04:02,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642625 2024-12-10T11:04:02,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642625 2024-12-10T11:04:02,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642625) roll requested 2024-12-10T11:04:02,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642631 2024-12-10T11:04:02,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642631 2024-12-10T11:04:02,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642631) roll requested 2024-12-10T11:04:02,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642641 2024-12-10T11:04:02,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642641 2024-12-10T11:04:02,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642641) roll requested 2024-12-10T11:04:02,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642647 2024-12-10T11:04:02,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642647 2024-12-10T11:04:02,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642647) roll requested 2024-12-10T11:04:02,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642653 2024-12-10T11:04:02,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642653 2024-12-10T11:04:02,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642653) roll requested 2024-12-10T11:04:02,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642660 2024-12-10T11:04:02,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642660 2024-12-10T11:04:02,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642660) roll requested 2024-12-10T11:04:02,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642667 2024-12-10T11:04:02,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642667 2024-12-10T11:04:02,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642667) roll requested 2024-12-10T11:04:02,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642674 2024-12-10T11:04:02,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642674 2024-12-10T11:04:02,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642674) roll requested 2024-12-10T11:04:02,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642680 2024-12-10T11:04:02,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642680 2024-12-10T11:04:02,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642680) roll requested 2024-12-10T11:04:02,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642686 2024-12-10T11:04:02,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642686 2024-12-10T11:04:02,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642686) roll requested 2024-12-10T11:04:02,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642693 2024-12-10T11:04:02,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642693 2024-12-10T11:04:02,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642693) roll requested 2024-12-10T11:04:02,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642699 2024-12-10T11:04:02,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642699 2024-12-10T11:04:02,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642699) roll requested 2024-12-10T11:04:02,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642705 2024-12-10T11:04:02,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642705 2024-12-10T11:04:02,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642705) roll requested 2024-12-10T11:04:02,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642711 2024-12-10T11:04:02,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642711 2024-12-10T11:04:02,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642711) roll requested 2024-12-10T11:04:02,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642718 2024-12-10T11:04:02,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642718 2024-12-10T11:04:02,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642718) roll requested 2024-12-10T11:04:02,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642724 2024-12-10T11:04:02,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642724 2024-12-10T11:04:02,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642724) roll requested 2024-12-10T11:04:02,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642730 2024-12-10T11:04:02,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642730 2024-12-10T11:04:02,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642730) roll requested 2024-12-10T11:04:02,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642736 2024-12-10T11:04:02,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642736 2024-12-10T11:04:02,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642736) roll requested 2024-12-10T11:04:02,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642742 2024-12-10T11:04:02,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642742 2024-12-10T11:04:02,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642742) roll requested 2024-12-10T11:04:02,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642748 2024-12-10T11:04:02,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642748 2024-12-10T11:04:02,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642748) roll requested 2024-12-10T11:04:02,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642754 2024-12-10T11:04:02,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642754 2024-12-10T11:04:02,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642754) roll requested 2024-12-10T11:04:02,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642760 2024-12-10T11:04:02,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642760 2024-12-10T11:04:02,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642760) roll requested 2024-12-10T11:04:02,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642767 2024-12-10T11:04:02,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642767 2024-12-10T11:04:02,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642767) roll requested 2024-12-10T11:04:02,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642773 2024-12-10T11:04:02,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642773 2024-12-10T11:04:02,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642773) roll requested 2024-12-10T11:04:02,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642779 2024-12-10T11:04:02,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642779 2024-12-10T11:04:02,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642779) roll requested 2024-12-10T11:04:02,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642785 2024-12-10T11:04:02,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642785 2024-12-10T11:04:02,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642785) roll requested 2024-12-10T11:04:02,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642791 2024-12-10T11:04:02,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642791 2024-12-10T11:04:02,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642791) roll requested 2024-12-10T11:04:02,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642797 2024-12-10T11:04:02,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642797 2024-12-10T11:04:02,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642797) roll requested 2024-12-10T11:04:02,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642803 2024-12-10T11:04:02,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642803 2024-12-10T11:04:02,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642803) roll requested 2024-12-10T11:04:02,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642809 2024-12-10T11:04:02,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642809 2024-12-10T11:04:02,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642809) roll requested 2024-12-10T11:04:02,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642815 2024-12-10T11:04:02,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642815 2024-12-10T11:04:02,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642815) roll requested 2024-12-10T11:04:02,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642821 2024-12-10T11:04:02,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642821 2024-12-10T11:04:02,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642821) roll requested 2024-12-10T11:04:02,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642827 2024-12-10T11:04:02,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642827 2024-12-10T11:04:02,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642827) roll requested 2024-12-10T11:04:02,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642833 2024-12-10T11:04:02,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642833 2024-12-10T11:04:02,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642833) roll requested 2024-12-10T11:04:02,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642841 2024-12-10T11:04:02,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642841 2024-12-10T11:04:02,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642841) roll requested 2024-12-10T11:04:02,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642847 2024-12-10T11:04:02,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642847 2024-12-10T11:04:02,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642847) roll requested 2024-12-10T11:04:02,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642853 2024-12-10T11:04:02,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642853 2024-12-10T11:04:02,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642853) roll requested 2024-12-10T11:04:02,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642860 2024-12-10T11:04:02,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642860 2024-12-10T11:04:02,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642860) roll requested 2024-12-10T11:04:02,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642866 2024-12-10T11:04:02,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642866 2024-12-10T11:04:02,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642866) roll requested 2024-12-10T11:04:02,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642872 2024-12-10T11:04:02,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642872 2024-12-10T11:04:02,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642872) roll requested 2024-12-10T11:04:02,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642878 2024-12-10T11:04:02,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642878 2024-12-10T11:04:02,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642878) roll requested 2024-12-10T11:04:02,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642885 2024-12-10T11:04:02,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642885 2024-12-10T11:04:02,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642885) roll requested 2024-12-10T11:04:02,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642891 2024-12-10T11:04:02,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642891 2024-12-10T11:04:02,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642891) roll requested 2024-12-10T11:04:02,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642897 2024-12-10T11:04:02,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642897 2024-12-10T11:04:02,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642897) roll requested 2024-12-10T11:04:02,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642903 2024-12-10T11:04:02,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642903 2024-12-10T11:04:02,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642903) roll requested 2024-12-10T11:04:02,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642909 2024-12-10T11:04:02,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642909 2024-12-10T11:04:02,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642909) roll requested 2024-12-10T11:04:02,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642915 2024-12-10T11:04:02,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642915 2024-12-10T11:04:02,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642915) roll requested 2024-12-10T11:04:02,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642921 2024-12-10T11:04:02,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642921 2024-12-10T11:04:02,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642921) roll requested 2024-12-10T11:04:02,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642927 2024-12-10T11:04:02,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642927 2024-12-10T11:04:02,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642927) roll requested 2024-12-10T11:04:02,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642933 2024-12-10T11:04:02,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642933 2024-12-10T11:04:02,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642933) roll requested 2024-12-10T11:04:02,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642939 2024-12-10T11:04:02,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642939 2024-12-10T11:04:02,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642939) roll requested 2024-12-10T11:04:02,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642945 2024-12-10T11:04:02,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642945 2024-12-10T11:04:02,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642945) roll requested 2024-12-10T11:04:02,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642951 2024-12-10T11:04:02,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642951 2024-12-10T11:04:02,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642951) roll requested 2024-12-10T11:04:02,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642958 2024-12-10T11:04:02,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642958 2024-12-10T11:04:02,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642958) roll requested 2024-12-10T11:04:02,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642964 2024-12-10T11:04:02,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642964 2024-12-10T11:04:02,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642964) roll requested 2024-12-10T11:04:02,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642973 2024-12-10T11:04:02,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642973 2024-12-10T11:04:02,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642973) roll requested 2024-12-10T11:04:02,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642980 2024-12-10T11:04:02,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642980 2024-12-10T11:04:02,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642980) roll requested 2024-12-10T11:04:02,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642991 2024-12-10T11:04:02,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:02,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642991 2024-12-10T11:04:02,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:02,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:02,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642991) roll requested 2024-12-10T11:04:02,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828642997 2024-12-10T11:04:03,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642997 2024-12-10T11:04:03,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828642997) roll requested 2024-12-10T11:04:03,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643003 2024-12-10T11:04:03,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828642997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643003 2024-12-10T11:04:03,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643003) roll requested 2024-12-10T11:04:03,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643009 2024-12-10T11:04:03,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643009 2024-12-10T11:04:03,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643009) roll requested 2024-12-10T11:04:03,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643017 2024-12-10T11:04:03,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643017 2024-12-10T11:04:03,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643017) roll requested 2024-12-10T11:04:03,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643026 2024-12-10T11:04:03,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643026 2024-12-10T11:04:03,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643026) roll requested 2024-12-10T11:04:03,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643033 2024-12-10T11:04:03,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643033 2024-12-10T11:04:03,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643033) roll requested 2024-12-10T11:04:03,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643043 2024-12-10T11:04:03,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643043 2024-12-10T11:04:03,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643043) roll requested 2024-12-10T11:04:03,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643049 2024-12-10T11:04:03,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643049 2024-12-10T11:04:03,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643049) roll requested 2024-12-10T11:04:03,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643057 2024-12-10T11:04:03,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643057 2024-12-10T11:04:03,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643057) roll requested 2024-12-10T11:04:03,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643064 2024-12-10T11:04:03,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643064 2024-12-10T11:04:03,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643064) roll requested 2024-12-10T11:04:03,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643073 2024-12-10T11:04:03,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643073 2024-12-10T11:04:03,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643073) roll requested 2024-12-10T11:04:03,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643079 2024-12-10T11:04:03,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643079 2024-12-10T11:04:03,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643079) roll requested 2024-12-10T11:04:03,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643086 2024-12-10T11:04:03,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643086 2024-12-10T11:04:03,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643086) roll requested 2024-12-10T11:04:03,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643092 2024-12-10T11:04:03,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643092 2024-12-10T11:04:03,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643092) roll requested 2024-12-10T11:04:03,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643101 2024-12-10T11:04:03,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643101 2024-12-10T11:04:03,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643101) roll requested 2024-12-10T11:04:03,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643107 2024-12-10T11:04:03,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643107 2024-12-10T11:04:03,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643107) roll requested 2024-12-10T11:04:03,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643112 2024-12-10T11:04:03,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643112 2024-12-10T11:04:03,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643112) roll requested 2024-12-10T11:04:03,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643119 2024-12-10T11:04:03,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643119 2024-12-10T11:04:03,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643119) roll requested 2024-12-10T11:04:03,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643125 2024-12-10T11:04:03,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643125 2024-12-10T11:04:03,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643125) roll requested 2024-12-10T11:04:03,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643131 2024-12-10T11:04:03,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643131 2024-12-10T11:04:03,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643131) roll requested 2024-12-10T11:04:03,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643137 2024-12-10T11:04:03,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643137 2024-12-10T11:04:03,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643137) roll requested 2024-12-10T11:04:03,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643143 2024-12-10T11:04:03,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643143 2024-12-10T11:04:03,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643143) roll requested 2024-12-10T11:04:03,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643149 2024-12-10T11:04:03,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643149 2024-12-10T11:04:03,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643149) roll requested 2024-12-10T11:04:03,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643155 2024-12-10T11:04:03,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643155 2024-12-10T11:04:03,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643155) roll requested 2024-12-10T11:04:03,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643161 2024-12-10T11:04:03,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643161 2024-12-10T11:04:03,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643161) roll requested 2024-12-10T11:04:03,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643167 2024-12-10T11:04:03,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643167 2024-12-10T11:04:03,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643167) roll requested 2024-12-10T11:04:03,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643173 2024-12-10T11:04:03,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643173 2024-12-10T11:04:03,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643173) roll requested 2024-12-10T11:04:03,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643179 2024-12-10T11:04:03,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643179 2024-12-10T11:04:03,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643179) roll requested 2024-12-10T11:04:03,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643185 2024-12-10T11:04:03,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643185 2024-12-10T11:04:03,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643185) roll requested 2024-12-10T11:04:03,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643191 2024-12-10T11:04:03,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643191 2024-12-10T11:04:03,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643191) roll requested 2024-12-10T11:04:03,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643197 2024-12-10T11:04:03,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643197 2024-12-10T11:04:03,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643197) roll requested 2024-12-10T11:04:03,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643203 2024-12-10T11:04:03,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643203 2024-12-10T11:04:03,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643203) roll requested 2024-12-10T11:04:03,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643209 2024-12-10T11:04:03,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643209 2024-12-10T11:04:03,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643209) roll requested 2024-12-10T11:04:03,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643214 2024-12-10T11:04:03,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643214 2024-12-10T11:04:03,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643214) roll requested 2024-12-10T11:04:03,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643220 2024-12-10T11:04:03,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643220 2024-12-10T11:04:03,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643220) roll requested 2024-12-10T11:04:03,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643227 2024-12-10T11:04:03,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643227 2024-12-10T11:04:03,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643227) roll requested 2024-12-10T11:04:03,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643232 2024-12-10T11:04:03,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643232 2024-12-10T11:04:03,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643232) roll requested 2024-12-10T11:04:03,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643238 2024-12-10T11:04:03,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643238 2024-12-10T11:04:03,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643238) roll requested 2024-12-10T11:04:03,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643244 2024-12-10T11:04:03,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643244 2024-12-10T11:04:03,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643244) roll requested 2024-12-10T11:04:03,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643250 2024-12-10T11:04:03,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643250 2024-12-10T11:04:03,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643250) roll requested 2024-12-10T11:04:03,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643256 2024-12-10T11:04:03,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643256 2024-12-10T11:04:03,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643256) roll requested 2024-12-10T11:04:03,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643262 2024-12-10T11:04:03,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643262 2024-12-10T11:04:03,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643262) roll requested 2024-12-10T11:04:03,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643268 2024-12-10T11:04:03,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643268 2024-12-10T11:04:03,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643268) roll requested 2024-12-10T11:04:03,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643274 2024-12-10T11:04:03,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643274 2024-12-10T11:04:03,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643274) roll requested 2024-12-10T11:04:03,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643280 2024-12-10T11:04:03,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643280 2024-12-10T11:04:03,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643280) roll requested 2024-12-10T11:04:03,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643286 2024-12-10T11:04:03,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643286 2024-12-10T11:04:03,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643286) roll requested 2024-12-10T11:04:03,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643292 2024-12-10T11:04:03,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643292 2024-12-10T11:04:03,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643292) roll requested 2024-12-10T11:04:03,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643298 2024-12-10T11:04:03,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643298 2024-12-10T11:04:03,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643298) roll requested 2024-12-10T11:04:03,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643304 2024-12-10T11:04:03,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643304 2024-12-10T11:04:03,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643304) roll requested 2024-12-10T11:04:03,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643311 2024-12-10T11:04:03,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643311 2024-12-10T11:04:03,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643311) roll requested 2024-12-10T11:04:03,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643317 2024-12-10T11:04:03,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643317 2024-12-10T11:04:03,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643317) roll requested 2024-12-10T11:04:03,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643323 2024-12-10T11:04:03,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643323 2024-12-10T11:04:03,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643323) roll requested 2024-12-10T11:04:03,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643328 2024-12-10T11:04:03,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643328 2024-12-10T11:04:03,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643328) roll requested 2024-12-10T11:04:03,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643335 2024-12-10T11:04:03,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643335 2024-12-10T11:04:03,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643335) roll requested 2024-12-10T11:04:03,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643341 2024-12-10T11:04:03,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643341 2024-12-10T11:04:03,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643341) roll requested 2024-12-10T11:04:03,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643347 2024-12-10T11:04:03,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643347 2024-12-10T11:04:03,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643347) roll requested 2024-12-10T11:04:03,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643353 2024-12-10T11:04:03,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643353 2024-12-10T11:04:03,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643353) roll requested 2024-12-10T11:04:03,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643359 2024-12-10T11:04:03,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643359 2024-12-10T11:04:03,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643359) roll requested 2024-12-10T11:04:03,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643365 2024-12-10T11:04:03,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643365 2024-12-10T11:04:03,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643365) roll requested 2024-12-10T11:04:03,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643371 2024-12-10T11:04:03,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643371 2024-12-10T11:04:03,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643371) roll requested 2024-12-10T11:04:03,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643377 2024-12-10T11:04:03,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643377 2024-12-10T11:04:03,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643377) roll requested 2024-12-10T11:04:03,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643384 2024-12-10T11:04:03,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643384 2024-12-10T11:04:03,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643384) roll requested 2024-12-10T11:04:03,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643390 2024-12-10T11:04:03,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643390 2024-12-10T11:04:03,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643390) roll requested 2024-12-10T11:04:03,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643396 2024-12-10T11:04:03,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643396 2024-12-10T11:04:03,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643396) roll requested 2024-12-10T11:04:03,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643402 2024-12-10T11:04:03,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643402 2024-12-10T11:04:03,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643402) roll requested 2024-12-10T11:04:03,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643409 2024-12-10T11:04:03,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643409 2024-12-10T11:04:03,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643409) roll requested 2024-12-10T11:04:03,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643415 2024-12-10T11:04:03,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643415 2024-12-10T11:04:03,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643415) roll requested 2024-12-10T11:04:03,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643421 2024-12-10T11:04:03,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643421 2024-12-10T11:04:03,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643421) roll requested 2024-12-10T11:04:03,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643427 2024-12-10T11:04:03,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643427 2024-12-10T11:04:03,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643427) roll requested 2024-12-10T11:04:03,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643433 2024-12-10T11:04:03,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643433 2024-12-10T11:04:03,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643433) roll requested 2024-12-10T11:04:03,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643439 2024-12-10T11:04:03,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643439 2024-12-10T11:04:03,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643439) roll requested 2024-12-10T11:04:03,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643445 2024-12-10T11:04:03,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643445 2024-12-10T11:04:03,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643445) roll requested 2024-12-10T11:04:03,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643457 2024-12-10T11:04:03,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643457 2024-12-10T11:04:03,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643457) roll requested 2024-12-10T11:04:03,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643464 2024-12-10T11:04:03,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643464 2024-12-10T11:04:03,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643464) roll requested 2024-12-10T11:04:03,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643473 2024-12-10T11:04:03,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643473 2024-12-10T11:04:03,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643473) roll requested 2024-12-10T11:04:03,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643481 2024-12-10T11:04:03,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643481 2024-12-10T11:04:03,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643481) roll requested 2024-12-10T11:04:03,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643488 2024-12-10T11:04:03,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643488 2024-12-10T11:04:03,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643488) roll requested 2024-12-10T11:04:03,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643497 2024-12-10T11:04:03,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643497 2024-12-10T11:04:03,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643497) roll requested 2024-12-10T11:04:03,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643505 2024-12-10T11:04:03,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643505 2024-12-10T11:04:03,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643505) roll requested 2024-12-10T11:04:03,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643511 2024-12-10T11:04:03,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643511 2024-12-10T11:04:03,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643511) roll requested 2024-12-10T11:04:03,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643517 2024-12-10T11:04:03,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643517 2024-12-10T11:04:03,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643517) roll requested 2024-12-10T11:04:03,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643524 2024-12-10T11:04:03,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643524 2024-12-10T11:04:03,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643524) roll requested 2024-12-10T11:04:03,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643530 2024-12-10T11:04:03,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643530 2024-12-10T11:04:03,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643530) roll requested 2024-12-10T11:04:03,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643536 2024-12-10T11:04:03,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643536 2024-12-10T11:04:03,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643536) roll requested 2024-12-10T11:04:03,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643542 2024-12-10T11:04:03,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643542 2024-12-10T11:04:03,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643542) roll requested 2024-12-10T11:04:03,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643549 2024-12-10T11:04:03,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643549 2024-12-10T11:04:03,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643549) roll requested 2024-12-10T11:04:03,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643555 2024-12-10T11:04:03,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643555 2024-12-10T11:04:03,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643555) roll requested 2024-12-10T11:04:03,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643561 2024-12-10T11:04:03,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643561 2024-12-10T11:04:03,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643561) roll requested 2024-12-10T11:04:03,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643568 2024-12-10T11:04:03,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643568 2024-12-10T11:04:03,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643568) roll requested 2024-12-10T11:04:03,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643574 2024-12-10T11:04:03,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643574 2024-12-10T11:04:03,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643574) roll requested 2024-12-10T11:04:03,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643580 2024-12-10T11:04:03,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643580 2024-12-10T11:04:03,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643580) roll requested 2024-12-10T11:04:03,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643587 2024-12-10T11:04:03,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643587 2024-12-10T11:04:03,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643587) roll requested 2024-12-10T11:04:03,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643593 2024-12-10T11:04:03,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643593 2024-12-10T11:04:03,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643593) roll requested 2024-12-10T11:04:03,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643600 2024-12-10T11:04:03,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643600 2024-12-10T11:04:03,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643600) roll requested 2024-12-10T11:04:03,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643607 2024-12-10T11:04:03,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643607 2024-12-10T11:04:03,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643607) roll requested 2024-12-10T11:04:03,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643614 2024-12-10T11:04:03,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643614 2024-12-10T11:04:03,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643614) roll requested 2024-12-10T11:04:03,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643620 2024-12-10T11:04:03,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643620 2024-12-10T11:04:03,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643620) roll requested 2024-12-10T11:04:03,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643626 2024-12-10T11:04:03,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643626 2024-12-10T11:04:03,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643626) roll requested 2024-12-10T11:04:03,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643632 2024-12-10T11:04:03,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643632 2024-12-10T11:04:03,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643632) roll requested 2024-12-10T11:04:03,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643639 2024-12-10T11:04:03,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643639 2024-12-10T11:04:03,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643639) roll requested 2024-12-10T11:04:03,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643645 2024-12-10T11:04:03,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643645 2024-12-10T11:04:03,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643645) roll requested 2024-12-10T11:04:03,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643652 2024-12-10T11:04:03,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643652 2024-12-10T11:04:03,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:03,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643652) roll requested 2024-12-10T11:04:03,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643658 2024-12-10T11:04:03,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643658 2024-12-10T11:04:03,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643658) roll requested 2024-12-10T11:04:03,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643671 2024-12-10T11:04:03,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643671 2024-12-10T11:04:03,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643671) roll requested 2024-12-10T11:04:03,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643679 2024-12-10T11:04:03,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643679 2024-12-10T11:04:03,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643679) roll requested 2024-12-10T11:04:03,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643691 2024-12-10T11:04:03,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643691 2024-12-10T11:04:03,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643691) roll requested 2024-12-10T11:04:03,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643699 2024-12-10T11:04:03,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643699 2024-12-10T11:04:03,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643699) roll requested 2024-12-10T11:04:03,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643710 2024-12-10T11:04:03,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643710 2024-12-10T11:04:03,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643710) roll requested 2024-12-10T11:04:03,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643717 2024-12-10T11:04:03,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643717 2024-12-10T11:04:03,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643717) roll requested 2024-12-10T11:04:03,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643724 2024-12-10T11:04:03,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643724 2024-12-10T11:04:03,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643724) roll requested 2024-12-10T11:04:03,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643731 2024-12-10T11:04:03,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643731 2024-12-10T11:04:03,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643731) roll requested 2024-12-10T11:04:03,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643738 2024-12-10T11:04:03,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643738 2024-12-10T11:04:03,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643738) roll requested 2024-12-10T11:04:03,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643745 2024-12-10T11:04:03,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643745 2024-12-10T11:04:03,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643745) roll requested 2024-12-10T11:04:03,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643754 2024-12-10T11:04:03,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643754 2024-12-10T11:04:03,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643754) roll requested 2024-12-10T11:04:03,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643761 2024-12-10T11:04:03,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643761 2024-12-10T11:04:03,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643761) roll requested 2024-12-10T11:04:03,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643769 2024-12-10T11:04:03,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643769 2024-12-10T11:04:03,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643769) roll requested 2024-12-10T11:04:03,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643779 2024-12-10T11:04:03,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643779 2024-12-10T11:04:03,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643779) roll requested 2024-12-10T11:04:03,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643786 2024-12-10T11:04:03,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643786 2024-12-10T11:04:03,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643786) roll requested 2024-12-10T11:04:03,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643794 2024-12-10T11:04:03,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643794 2024-12-10T11:04:03,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643794) roll requested 2024-12-10T11:04:03,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643800 2024-12-10T11:04:03,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643800 2024-12-10T11:04:03,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643800) roll requested 2024-12-10T11:04:03,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643806 2024-12-10T11:04:03,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643806 2024-12-10T11:04:03,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643806) roll requested 2024-12-10T11:04:03,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643813 2024-12-10T11:04:03,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643813 2024-12-10T11:04:03,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643813) roll requested 2024-12-10T11:04:03,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643819 2024-12-10T11:04:03,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643819 2024-12-10T11:04:03,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643819) roll requested 2024-12-10T11:04:03,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643826 2024-12-10T11:04:03,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643826 2024-12-10T11:04:03,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643826) roll requested 2024-12-10T11:04:03,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643832 2024-12-10T11:04:03,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643826 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643832 2024-12-10T11:04:03,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643832) roll requested 2024-12-10T11:04:03,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643839 2024-12-10T11:04:03,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643839 2024-12-10T11:04:03,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643839) roll requested 2024-12-10T11:04:03,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643845 2024-12-10T11:04:03,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643845 2024-12-10T11:04:03,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643845) roll requested 2024-12-10T11:04:03,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643851 2024-12-10T11:04:03,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643851 2024-12-10T11:04:03,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643851) roll requested 2024-12-10T11:04:03,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643858 2024-12-10T11:04:03,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643858 2024-12-10T11:04:03,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643858) roll requested 2024-12-10T11:04:03,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643864 2024-12-10T11:04:03,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643864 2024-12-10T11:04:03,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643864) roll requested 2024-12-10T11:04:03,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643870 2024-12-10T11:04:03,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643870 2024-12-10T11:04:03,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643870) roll requested 2024-12-10T11:04:03,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643877 2024-12-10T11:04:03,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643877 2024-12-10T11:04:03,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643877) roll requested 2024-12-10T11:04:03,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643883 2024-12-10T11:04:03,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643883 2024-12-10T11:04:03,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643883) roll requested 2024-12-10T11:04:03,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643890 2024-12-10T11:04:03,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643890 2024-12-10T11:04:03,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643890) roll requested 2024-12-10T11:04:03,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643897 2024-12-10T11:04:03,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643897 2024-12-10T11:04:03,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643897) roll requested 2024-12-10T11:04:03,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643903 2024-12-10T11:04:03,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643903 2024-12-10T11:04:03,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643903) roll requested 2024-12-10T11:04:03,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643909 2024-12-10T11:04:03,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643909 2024-12-10T11:04:03,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643909) roll requested 2024-12-10T11:04:03,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643916 2024-12-10T11:04:03,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643916 2024-12-10T11:04:03,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643916) roll requested 2024-12-10T11:04:03,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643922 2024-12-10T11:04:03,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643922 2024-12-10T11:04:03,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643922) roll requested 2024-12-10T11:04:03,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643928 2024-12-10T11:04:03,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643928 2024-12-10T11:04:03,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643928) roll requested 2024-12-10T11:04:03,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643934 2024-12-10T11:04:03,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643934 2024-12-10T11:04:03,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643934) roll requested 2024-12-10T11:04:03,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643942 2024-12-10T11:04:03,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643942 2024-12-10T11:04:03,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643942) roll requested 2024-12-10T11:04:03,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643948 2024-12-10T11:04:03,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643948 2024-12-10T11:04:03,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643948) roll requested 2024-12-10T11:04:03,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643954 2024-12-10T11:04:03,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643954 2024-12-10T11:04:03,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643954) roll requested 2024-12-10T11:04:03,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643961 2024-12-10T11:04:03,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643961 2024-12-10T11:04:03,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643961) roll requested 2024-12-10T11:04:03,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643967 2024-12-10T11:04:03,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643967 2024-12-10T11:04:03,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643967) roll requested 2024-12-10T11:04:03,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643974 2024-12-10T11:04:03,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643974 2024-12-10T11:04:03,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643974) roll requested 2024-12-10T11:04:03,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643987 2024-12-10T11:04:03,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:03,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643987 2024-12-10T11:04:03,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:03,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:03,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643987) roll requested 2024-12-10T11:04:03,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828643994 2024-12-10T11:04:04,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643994 2024-12-10T11:04:04,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828643994) roll requested 2024-12-10T11:04:04,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644002 2024-12-10T11:04:04,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828643994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644002 2024-12-10T11:04:04,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644002) roll requested 2024-12-10T11:04:04,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644009 2024-12-10T11:04:04,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644009 2024-12-10T11:04:04,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644009) roll requested 2024-12-10T11:04:04,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644015 2024-12-10T11:04:04,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644015 2024-12-10T11:04:04,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644015) roll requested 2024-12-10T11:04:04,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644022 2024-12-10T11:04:04,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644022 2024-12-10T11:04:04,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644022) roll requested 2024-12-10T11:04:04,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644028 2024-12-10T11:04:04,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644028 2024-12-10T11:04:04,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644028) roll requested 2024-12-10T11:04:04,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644034 2024-12-10T11:04:04,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644034 2024-12-10T11:04:04,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644034) roll requested 2024-12-10T11:04:04,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644041 2024-12-10T11:04:04,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644041 2024-12-10T11:04:04,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644041) roll requested 2024-12-10T11:04:04,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644047 2024-12-10T11:04:04,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644047 2024-12-10T11:04:04,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644047) roll requested 2024-12-10T11:04:04,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644055 2024-12-10T11:04:04,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644055 2024-12-10T11:04:04,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644055) roll requested 2024-12-10T11:04:04,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644063 2024-12-10T11:04:04,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644063 2024-12-10T11:04:04,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644063) roll requested 2024-12-10T11:04:04,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644069 2024-12-10T11:04:04,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644069 2024-12-10T11:04:04,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644069) roll requested 2024-12-10T11:04:04,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644076 2024-12-10T11:04:04,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644076 2024-12-10T11:04:04,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644076) roll requested 2024-12-10T11:04:04,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644082 2024-12-10T11:04:04,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644082 2024-12-10T11:04:04,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644082) roll requested 2024-12-10T11:04:04,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644090 2024-12-10T11:04:04,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644090 2024-12-10T11:04:04,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644090) roll requested 2024-12-10T11:04:04,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644097 2024-12-10T11:04:04,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644097 2024-12-10T11:04:04,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644097) roll requested 2024-12-10T11:04:04,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644103 2024-12-10T11:04:04,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644103 2024-12-10T11:04:04,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644103) roll requested 2024-12-10T11:04:04,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644110 2024-12-10T11:04:04,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644110 2024-12-10T11:04:04,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644110) roll requested 2024-12-10T11:04:04,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644116 2024-12-10T11:04:04,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644116 2024-12-10T11:04:04,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644116) roll requested 2024-12-10T11:04:04,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644122 2024-12-10T11:04:04,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644122 2024-12-10T11:04:04,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644122) roll requested 2024-12-10T11:04:04,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644129 2024-12-10T11:04:04,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644129 2024-12-10T11:04:04,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644129) roll requested 2024-12-10T11:04:04,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644135 2024-12-10T11:04:04,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644135 2024-12-10T11:04:04,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644135) roll requested 2024-12-10T11:04:04,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644141 2024-12-10T11:04:04,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644141 2024-12-10T11:04:04,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644141) roll requested 2024-12-10T11:04:04,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644148 2024-12-10T11:04:04,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644148 2024-12-10T11:04:04,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644148) roll requested 2024-12-10T11:04:04,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644154 2024-12-10T11:04:04,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644154 2024-12-10T11:04:04,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644154) roll requested 2024-12-10T11:04:04,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644161 2024-12-10T11:04:04,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644161 2024-12-10T11:04:04,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644161) roll requested 2024-12-10T11:04:04,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644168 2024-12-10T11:04:04,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644168 2024-12-10T11:04:04,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644168) roll requested 2024-12-10T11:04:04,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644174 2024-12-10T11:04:04,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644174 2024-12-10T11:04:04,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644174) roll requested 2024-12-10T11:04:04,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644180 2024-12-10T11:04:04,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644180 2024-12-10T11:04:04,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644180) roll requested 2024-12-10T11:04:04,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644187 2024-12-10T11:04:04,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644187 2024-12-10T11:04:04,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644187) roll requested 2024-12-10T11:04:04,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644196 2024-12-10T11:04:04,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644196 2024-12-10T11:04:04,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644196) roll requested 2024-12-10T11:04:04,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644203 2024-12-10T11:04:04,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644203 2024-12-10T11:04:04,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644203) roll requested 2024-12-10T11:04:04,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644210 2024-12-10T11:04:04,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644210 2024-12-10T11:04:04,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644210) roll requested 2024-12-10T11:04:04,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644216 2024-12-10T11:04:04,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644216 2024-12-10T11:04:04,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644216) roll requested 2024-12-10T11:04:04,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644223 2024-12-10T11:04:04,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644223 2024-12-10T11:04:04,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644223) roll requested 2024-12-10T11:04:04,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644233 2024-12-10T11:04:04,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644233 2024-12-10T11:04:04,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644233) roll requested 2024-12-10T11:04:04,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644240 2024-12-10T11:04:04,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644240 2024-12-10T11:04:04,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644240) roll requested 2024-12-10T11:04:04,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644247 2024-12-10T11:04:04,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644247 2024-12-10T11:04:04,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644247) roll requested 2024-12-10T11:04:04,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644254 2024-12-10T11:04:04,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644254 2024-12-10T11:04:04,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644254) roll requested 2024-12-10T11:04:04,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644261 2024-12-10T11:04:04,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644261 2024-12-10T11:04:04,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644261) roll requested 2024-12-10T11:04:04,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644268 2024-12-10T11:04:04,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644268 2024-12-10T11:04:04,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644268) roll requested 2024-12-10T11:04:04,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644275 2024-12-10T11:04:04,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644275 2024-12-10T11:04:04,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644275) roll requested 2024-12-10T11:04:04,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644282 2024-12-10T11:04:04,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644282 2024-12-10T11:04:04,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644282) roll requested 2024-12-10T11:04:04,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644288 2024-12-10T11:04:04,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644288 2024-12-10T11:04:04,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644288) roll requested 2024-12-10T11:04:04,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644294 2024-12-10T11:04:04,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644294 2024-12-10T11:04:04,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644294) roll requested 2024-12-10T11:04:04,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644301 2024-12-10T11:04:04,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644301 2024-12-10T11:04:04,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644301) roll requested 2024-12-10T11:04:04,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644307 2024-12-10T11:04:04,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644307 2024-12-10T11:04:04,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644307) roll requested 2024-12-10T11:04:04,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644313 2024-12-10T11:04:04,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644313 2024-12-10T11:04:04,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644313) roll requested 2024-12-10T11:04:04,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644319 2024-12-10T11:04:04,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644319 2024-12-10T11:04:04,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644319) roll requested 2024-12-10T11:04:04,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644326 2024-12-10T11:04:04,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644326 2024-12-10T11:04:04,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644326) roll requested 2024-12-10T11:04:04,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644332 2024-12-10T11:04:04,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644332 2024-12-10T11:04:04,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644332) roll requested 2024-12-10T11:04:04,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644339 2024-12-10T11:04:04,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644339 2024-12-10T11:04:04,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644339) roll requested 2024-12-10T11:04:04,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644345 2024-12-10T11:04:04,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644345 2024-12-10T11:04:04,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644345) roll requested 2024-12-10T11:04:04,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644352 2024-12-10T11:04:04,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644352 2024-12-10T11:04:04,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644352) roll requested 2024-12-10T11:04:04,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644358 2024-12-10T11:04:04,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644358 2024-12-10T11:04:04,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644358) roll requested 2024-12-10T11:04:04,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644364 2024-12-10T11:04:04,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644364 2024-12-10T11:04:04,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644364) roll requested 2024-12-10T11:04:04,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644370 2024-12-10T11:04:04,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644370 2024-12-10T11:04:04,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644370) roll requested 2024-12-10T11:04:04,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644377 2024-12-10T11:04:04,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644377 2024-12-10T11:04:04,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644377) roll requested 2024-12-10T11:04:04,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644390 2024-12-10T11:04:04,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644390 2024-12-10T11:04:04,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644390) roll requested 2024-12-10T11:04:04,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644401 2024-12-10T11:04:04,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644401 2024-12-10T11:04:04,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644401) roll requested 2024-12-10T11:04:04,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644408 2024-12-10T11:04:04,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644408 2024-12-10T11:04:04,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644408) roll requested 2024-12-10T11:04:04,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644414 2024-12-10T11:04:04,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644414 2024-12-10T11:04:04,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644414) roll requested 2024-12-10T11:04:04,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644420 2024-12-10T11:04:04,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644420 2024-12-10T11:04:04,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644420) roll requested 2024-12-10T11:04:04,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644427 2024-12-10T11:04:04,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644427 2024-12-10T11:04:04,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644427) roll requested 2024-12-10T11:04:04,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644434 2024-12-10T11:04:04,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644434 2024-12-10T11:04:04,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644434) roll requested 2024-12-10T11:04:04,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644440 2024-12-10T11:04:04,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644440 2024-12-10T11:04:04,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644440) roll requested 2024-12-10T11:04:04,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644446 2024-12-10T11:04:04,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644446 2024-12-10T11:04:04,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644446) roll requested 2024-12-10T11:04:04,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644453 2024-12-10T11:04:04,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644453 2024-12-10T11:04:04,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644453) roll requested 2024-12-10T11:04:04,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644459 2024-12-10T11:04:04,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644459 2024-12-10T11:04:04,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644459) roll requested 2024-12-10T11:04:04,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644466 2024-12-10T11:04:04,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644466 2024-12-10T11:04:04,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644466) roll requested 2024-12-10T11:04:04,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644472 2024-12-10T11:04:04,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644472 2024-12-10T11:04:04,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644472) roll requested 2024-12-10T11:04:04,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644478 2024-12-10T11:04:04,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644478 2024-12-10T11:04:04,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644478) roll requested 2024-12-10T11:04:04,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644485 2024-12-10T11:04:04,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644485 2024-12-10T11:04:04,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644485) roll requested 2024-12-10T11:04:04,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644493 2024-12-10T11:04:04,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644493 2024-12-10T11:04:04,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644493) roll requested 2024-12-10T11:04:04,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644499 2024-12-10T11:04:04,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644499 2024-12-10T11:04:04,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644499) roll requested 2024-12-10T11:04:04,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644505 2024-12-10T11:04:04,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644505 2024-12-10T11:04:04,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644505) roll requested 2024-12-10T11:04:04,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644512 2024-12-10T11:04:04,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644512 2024-12-10T11:04:04,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644512) roll requested 2024-12-10T11:04:04,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644518 2024-12-10T11:04:04,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644518 2024-12-10T11:04:04,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644518) roll requested 2024-12-10T11:04:04,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644525 2024-12-10T11:04:04,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644525 2024-12-10T11:04:04,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644525) roll requested 2024-12-10T11:04:04,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644531 2024-12-10T11:04:04,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644531 2024-12-10T11:04:04,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644531) roll requested 2024-12-10T11:04:04,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644538 2024-12-10T11:04:04,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644538 2024-12-10T11:04:04,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644538) roll requested 2024-12-10T11:04:04,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644544 2024-12-10T11:04:04,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644544 2024-12-10T11:04:04,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644544) roll requested 2024-12-10T11:04:04,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644551 2024-12-10T11:04:04,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644551 2024-12-10T11:04:04,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644551) roll requested 2024-12-10T11:04:04,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644558 2024-12-10T11:04:04,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644558 2024-12-10T11:04:04,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644558) roll requested 2024-12-10T11:04:04,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644564 2024-12-10T11:04:04,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644564 2024-12-10T11:04:04,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644564) roll requested 2024-12-10T11:04:04,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644571 2024-12-10T11:04:04,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644571 2024-12-10T11:04:04,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644571) roll requested 2024-12-10T11:04:04,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644577 2024-12-10T11:04:04,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644577 2024-12-10T11:04:04,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644577) roll requested 2024-12-10T11:04:04,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644583 2024-12-10T11:04:04,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644583 2024-12-10T11:04:04,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644583) roll requested 2024-12-10T11:04:04,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644590 2024-12-10T11:04:04,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644590 2024-12-10T11:04:04,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644590) roll requested 2024-12-10T11:04:04,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644596 2024-12-10T11:04:04,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644596 2024-12-10T11:04:04,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644596) roll requested 2024-12-10T11:04:04,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644603 2024-12-10T11:04:04,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644603 2024-12-10T11:04:04,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644603) roll requested 2024-12-10T11:04:04,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644609 2024-12-10T11:04:04,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644609 2024-12-10T11:04:04,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644609) roll requested 2024-12-10T11:04:04,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644616 2024-12-10T11:04:04,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644616 2024-12-10T11:04:04,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644616) roll requested 2024-12-10T11:04:04,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644624 2024-12-10T11:04:04,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644624 2024-12-10T11:04:04,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644624) roll requested 2024-12-10T11:04:04,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644630 2024-12-10T11:04:04,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644630 2024-12-10T11:04:04,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644630) roll requested 2024-12-10T11:04:04,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644637 2024-12-10T11:04:04,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644637 2024-12-10T11:04:04,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,643 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644637) roll requested 2024-12-10T11:04:04,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644643 2024-12-10T11:04:04,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644643 2024-12-10T11:04:04,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644643) roll requested 2024-12-10T11:04:04,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644649 2024-12-10T11:04:04,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644649 2024-12-10T11:04:04,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644649) roll requested 2024-12-10T11:04:04,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644656 2024-12-10T11:04:04,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644656 2024-12-10T11:04:04,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644656) roll requested 2024-12-10T11:04:04,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644662 2024-12-10T11:04:04,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644662 2024-12-10T11:04:04,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644662) roll requested 2024-12-10T11:04:04,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644668 2024-12-10T11:04:04,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644668 2024-12-10T11:04:04,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644668) roll requested 2024-12-10T11:04:04,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644675 2024-12-10T11:04:04,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644675 2024-12-10T11:04:04,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644675) roll requested 2024-12-10T11:04:04,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644682 2024-12-10T11:04:04,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644682 2024-12-10T11:04:04,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644682) roll requested 2024-12-10T11:04:04,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644689 2024-12-10T11:04:04,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644689 2024-12-10T11:04:04,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644689) roll requested 2024-12-10T11:04:04,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644695 2024-12-10T11:04:04,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644695 2024-12-10T11:04:04,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644695) roll requested 2024-12-10T11:04:04,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644702 2024-12-10T11:04:04,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644702 2024-12-10T11:04:04,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644702) roll requested 2024-12-10T11:04:04,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644708 2024-12-10T11:04:04,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644708 2024-12-10T11:04:04,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644708) roll requested 2024-12-10T11:04:04,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644714 2024-12-10T11:04:04,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644714 2024-12-10T11:04:04,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644714) roll requested 2024-12-10T11:04:04,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644721 2024-12-10T11:04:04,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644721 2024-12-10T11:04:04,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,727 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644721) roll requested 2024-12-10T11:04:04,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644727 2024-12-10T11:04:04,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644727 2024-12-10T11:04:04,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644727) roll requested 2024-12-10T11:04:04,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644733 2024-12-10T11:04:04,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644733 2024-12-10T11:04:04,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644733) roll requested 2024-12-10T11:04:04,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644739 2024-12-10T11:04:04,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644739 2024-12-10T11:04:04,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644739) roll requested 2024-12-10T11:04:04,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644746 2024-12-10T11:04:04,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644746 2024-12-10T11:04:04,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644746) roll requested 2024-12-10T11:04:04,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644752 2024-12-10T11:04:04,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644752 2024-12-10T11:04:04,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644752) roll requested 2024-12-10T11:04:04,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644758 2024-12-10T11:04:04,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644758 2024-12-10T11:04:04,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644758) roll requested 2024-12-10T11:04:04,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644765 2024-12-10T11:04:04,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644765 2024-12-10T11:04:04,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644765) roll requested 2024-12-10T11:04:04,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644772 2024-12-10T11:04:04,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644772 2024-12-10T11:04:04,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644772) roll requested 2024-12-10T11:04:04,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644778 2024-12-10T11:04:04,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644778 2024-12-10T11:04:04,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644778) roll requested 2024-12-10T11:04:04,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644784 2024-12-10T11:04:04,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644784 2024-12-10T11:04:04,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,790 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644784) roll requested 2024-12-10T11:04:04,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644790 2024-12-10T11:04:04,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644790 2024-12-10T11:04:04,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644790) roll requested 2024-12-10T11:04:04,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644796 2024-12-10T11:04:04,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644796 2024-12-10T11:04:04,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644796) roll requested 2024-12-10T11:04:04,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644803 2024-12-10T11:04:04,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644803 2024-12-10T11:04:04,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644803) roll requested 2024-12-10T11:04:04,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644809 2024-12-10T11:04:04,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644809 2024-12-10T11:04:04,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644809) roll requested 2024-12-10T11:04:04,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644815 2024-12-10T11:04:04,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644815 2024-12-10T11:04:04,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644815) roll requested 2024-12-10T11:04:04,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644822 2024-12-10T11:04:04,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644822 2024-12-10T11:04:04,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644822) roll requested 2024-12-10T11:04:04,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644829 2024-12-10T11:04:04,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644829 2024-12-10T11:04:04,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644829) roll requested 2024-12-10T11:04:04,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644835 2024-12-10T11:04:04,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644835 2024-12-10T11:04:04,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644835) roll requested 2024-12-10T11:04:04,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644841 2024-12-10T11:04:04,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644841 2024-12-10T11:04:04,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644841) roll requested 2024-12-10T11:04:04,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644848 2024-12-10T11:04:04,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644848 2024-12-10T11:04:04,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644848) roll requested 2024-12-10T11:04:04,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644854 2024-12-10T11:04:04,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644854 2024-12-10T11:04:04,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644854) roll requested 2024-12-10T11:04:04,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644860 2024-12-10T11:04:04,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644860 2024-12-10T11:04:04,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644860) roll requested 2024-12-10T11:04:04,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644867 2024-12-10T11:04:04,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644867 2024-12-10T11:04:04,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644867) roll requested 2024-12-10T11:04:04,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644873 2024-12-10T11:04:04,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644873 2024-12-10T11:04:04,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644873) roll requested 2024-12-10T11:04:04,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644880 2024-12-10T11:04:04,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644880 2024-12-10T11:04:04,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644880) roll requested 2024-12-10T11:04:04,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644887 2024-12-10T11:04:04,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644887 2024-12-10T11:04:04,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644887) roll requested 2024-12-10T11:04:04,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644893 2024-12-10T11:04:04,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644893 2024-12-10T11:04:04,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644893) roll requested 2024-12-10T11:04:04,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644900 2024-12-10T11:04:04,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644900 2024-12-10T11:04:04,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644900) roll requested 2024-12-10T11:04:04,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644906 2024-12-10T11:04:04,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644906 2024-12-10T11:04:04,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644906) roll requested 2024-12-10T11:04:04,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644912 2024-12-10T11:04:04,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644912 2024-12-10T11:04:04,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644912) roll requested 2024-12-10T11:04:04,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644919 2024-12-10T11:04:04,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644919 2024-12-10T11:04:04,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644919) roll requested 2024-12-10T11:04:04,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644925 2024-12-10T11:04:04,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644925 2024-12-10T11:04:04,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644925) roll requested 2024-12-10T11:04:04,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644933 2024-12-10T11:04:04,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644933 2024-12-10T11:04:04,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644933) roll requested 2024-12-10T11:04:04,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644944 2024-12-10T11:04:04,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644944 2024-12-10T11:04:04,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644944) roll requested 2024-12-10T11:04:04,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644951 2024-12-10T11:04:04,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644951 2024-12-10T11:04:04,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644951) roll requested 2024-12-10T11:04:04,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644958 2024-12-10T11:04:04,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644958 2024-12-10T11:04:04,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644958) roll requested 2024-12-10T11:04:04,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644964 2024-12-10T11:04:04,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644964 2024-12-10T11:04:04,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644964) roll requested 2024-12-10T11:04:04,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644971 2024-12-10T11:04:04,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644971 2024-12-10T11:04:04,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644971) roll requested 2024-12-10T11:04:04,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644977 2024-12-10T11:04:04,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644977 2024-12-10T11:04:04,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644977) roll requested 2024-12-10T11:04:04,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644983 2024-12-10T11:04:04,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644983 2024-12-10T11:04:04,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,990 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644983) roll requested 2024-12-10T11:04:04,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644990 2024-12-10T11:04:04,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:04,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644990 2024-12-10T11:04:04,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:04,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:04,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:04,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:04,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644990) roll requested 2024-12-10T11:04:04,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828644996 2024-12-10T11:04:05,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644996 2024-12-10T11:04:05,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828644996) roll requested 2024-12-10T11:04:05,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645002 2024-12-10T11:04:05,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828644996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645002 2024-12-10T11:04:05,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645002) roll requested 2024-12-10T11:04:05,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645009 2024-12-10T11:04:05,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645009 2024-12-10T11:04:05,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645009) roll requested 2024-12-10T11:04:05,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645015 2024-12-10T11:04:05,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645015 2024-12-10T11:04:05,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645015) roll requested 2024-12-10T11:04:05,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645022 2024-12-10T11:04:05,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645022 2024-12-10T11:04:05,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645022) roll requested 2024-12-10T11:04:05,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645028 2024-12-10T11:04:05,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645028 2024-12-10T11:04:05,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645028) roll requested 2024-12-10T11:04:05,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645035 2024-12-10T11:04:05,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645035 2024-12-10T11:04:05,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645035) roll requested 2024-12-10T11:04:05,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645041 2024-12-10T11:04:05,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645041 2024-12-10T11:04:05,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645041) roll requested 2024-12-10T11:04:05,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645048 2024-12-10T11:04:05,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645048 2024-12-10T11:04:05,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645048) roll requested 2024-12-10T11:04:05,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645054 2024-12-10T11:04:05,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645054 2024-12-10T11:04:05,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645054) roll requested 2024-12-10T11:04:05,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645060 2024-12-10T11:04:05,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645060 2024-12-10T11:04:05,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645060) roll requested 2024-12-10T11:04:05,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645067 2024-12-10T11:04:05,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645067 2024-12-10T11:04:05,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645067) roll requested 2024-12-10T11:04:05,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645073 2024-12-10T11:04:05,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645073 2024-12-10T11:04:05,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645073) roll requested 2024-12-10T11:04:05,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645080 2024-12-10T11:04:05,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645080 2024-12-10T11:04:05,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645080) roll requested 2024-12-10T11:04:05,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645086 2024-12-10T11:04:05,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645086 2024-12-10T11:04:05,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645086) roll requested 2024-12-10T11:04:05,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645093 2024-12-10T11:04:05,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645093 2024-12-10T11:04:05,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645093) roll requested 2024-12-10T11:04:05,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645099 2024-12-10T11:04:05,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645099 2024-12-10T11:04:05,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645099) roll requested 2024-12-10T11:04:05,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645106 2024-12-10T11:04:05,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645106 2024-12-10T11:04:05,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645106) roll requested 2024-12-10T11:04:05,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645114 2024-12-10T11:04:05,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645114 2024-12-10T11:04:05,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645114) roll requested 2024-12-10T11:04:05,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645120 2024-12-10T11:04:05,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645120 2024-12-10T11:04:05,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645120) roll requested 2024-12-10T11:04:05,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645127 2024-12-10T11:04:05,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645127 2024-12-10T11:04:05,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645127) roll requested 2024-12-10T11:04:05,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645133 2024-12-10T11:04:05,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645133 2024-12-10T11:04:05,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645133) roll requested 2024-12-10T11:04:05,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645139 2024-12-10T11:04:05,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645139 2024-12-10T11:04:05,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645139) roll requested 2024-12-10T11:04:05,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645146 2024-12-10T11:04:05,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645146 2024-12-10T11:04:05,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645146) roll requested 2024-12-10T11:04:05,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645152 2024-12-10T11:04:05,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645152 2024-12-10T11:04:05,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645152) roll requested 2024-12-10T11:04:05,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645159 2024-12-10T11:04:05,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645159 2024-12-10T11:04:05,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645159) roll requested 2024-12-10T11:04:05,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645165 2024-12-10T11:04:05,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645165 2024-12-10T11:04:05,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645165) roll requested 2024-12-10T11:04:05,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645171 2024-12-10T11:04:05,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645171 2024-12-10T11:04:05,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645171) roll requested 2024-12-10T11:04:05,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645178 2024-12-10T11:04:05,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645178 2024-12-10T11:04:05,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645178) roll requested 2024-12-10T11:04:05,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645185 2024-12-10T11:04:05,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645185 2024-12-10T11:04:05,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645185) roll requested 2024-12-10T11:04:05,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645194 2024-12-10T11:04:05,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645194 2024-12-10T11:04:05,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645194) roll requested 2024-12-10T11:04:05,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645200 2024-12-10T11:04:05,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645200 2024-12-10T11:04:05,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645200) roll requested 2024-12-10T11:04:05,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645211 2024-12-10T11:04:05,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645211 2024-12-10T11:04:05,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645211) roll requested 2024-12-10T11:04:05,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645218 2024-12-10T11:04:05,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645218 2024-12-10T11:04:05,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645218) roll requested 2024-12-10T11:04:05,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645227 2024-12-10T11:04:05,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645227 2024-12-10T11:04:05,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645227) roll requested 2024-12-10T11:04:05,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645234 2024-12-10T11:04:05,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645234 2024-12-10T11:04:05,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645234) roll requested 2024-12-10T11:04:05,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645241 2024-12-10T11:04:05,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645241 2024-12-10T11:04:05,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645241) roll requested 2024-12-10T11:04:05,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645247 2024-12-10T11:04:05,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645247 2024-12-10T11:04:05,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645247) roll requested 2024-12-10T11:04:05,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645253 2024-12-10T11:04:05,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645253 2024-12-10T11:04:05,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645253) roll requested 2024-12-10T11:04:05,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645260 2024-12-10T11:04:05,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645260 2024-12-10T11:04:05,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645260) roll requested 2024-12-10T11:04:05,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645266 2024-12-10T11:04:05,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645266 2024-12-10T11:04:05,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645266) roll requested 2024-12-10T11:04:05,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645272 2024-12-10T11:04:05,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645272 2024-12-10T11:04:05,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645272) roll requested 2024-12-10T11:04:05,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645279 2024-12-10T11:04:05,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645279 2024-12-10T11:04:05,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645279) roll requested 2024-12-10T11:04:05,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645286 2024-12-10T11:04:05,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645286 2024-12-10T11:04:05,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645286) roll requested 2024-12-10T11:04:05,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645293 2024-12-10T11:04:05,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645293 2024-12-10T11:04:05,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645293) roll requested 2024-12-10T11:04:05,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645299 2024-12-10T11:04:05,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645299 2024-12-10T11:04:05,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645299) roll requested 2024-12-10T11:04:05,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645305 2024-12-10T11:04:05,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645305 2024-12-10T11:04:05,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645305) roll requested 2024-12-10T11:04:05,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645312 2024-12-10T11:04:05,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645312 2024-12-10T11:04:05,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645312) roll requested 2024-12-10T11:04:05,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645318 2024-12-10T11:04:05,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645318 2024-12-10T11:04:05,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645318) roll requested 2024-12-10T11:04:05,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645324 2024-12-10T11:04:05,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645324 2024-12-10T11:04:05,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,330 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645324) roll requested 2024-12-10T11:04:05,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645330 2024-12-10T11:04:05,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645330 2024-12-10T11:04:05,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645330) roll requested 2024-12-10T11:04:05,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645337 2024-12-10T11:04:05,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645330 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645337 2024-12-10T11:04:05,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645337) roll requested 2024-12-10T11:04:05,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645343 2024-12-10T11:04:05,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645343 2024-12-10T11:04:05,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645343) roll requested 2024-12-10T11:04:05,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645350 2024-12-10T11:04:05,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645350 2024-12-10T11:04:05,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645350) roll requested 2024-12-10T11:04:05,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645356 2024-12-10T11:04:05,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645356 2024-12-10T11:04:05,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645356) roll requested 2024-12-10T11:04:05,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645362 2024-12-10T11:04:05,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645362 2024-12-10T11:04:05,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645362) roll requested 2024-12-10T11:04:05,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645368 2024-12-10T11:04:05,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645368 2024-12-10T11:04:05,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645368) roll requested 2024-12-10T11:04:05,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645375 2024-12-10T11:04:05,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645375 2024-12-10T11:04:05,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645375) roll requested 2024-12-10T11:04:05,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645381 2024-12-10T11:04:05,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645381 2024-12-10T11:04:05,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645381) roll requested 2024-12-10T11:04:05,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645388 2024-12-10T11:04:05,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645388 2024-12-10T11:04:05,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645388) roll requested 2024-12-10T11:04:05,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645394 2024-12-10T11:04:05,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645394 2024-12-10T11:04:05,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645394) roll requested 2024-12-10T11:04:05,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645401 2024-12-10T11:04:05,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645401 2024-12-10T11:04:05,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645401) roll requested 2024-12-10T11:04:05,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645407 2024-12-10T11:04:05,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645407 2024-12-10T11:04:05,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645407) roll requested 2024-12-10T11:04:05,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645414 2024-12-10T11:04:05,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645414 2024-12-10T11:04:05,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645414) roll requested 2024-12-10T11:04:05,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645420 2024-12-10T11:04:05,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645420 2024-12-10T11:04:05,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645420) roll requested 2024-12-10T11:04:05,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645426 2024-12-10T11:04:05,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645426 2024-12-10T11:04:05,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645426) roll requested 2024-12-10T11:04:05,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645432 2024-12-10T11:04:05,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645432 2024-12-10T11:04:05,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645432) roll requested 2024-12-10T11:04:05,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645439 2024-12-10T11:04:05,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645439 2024-12-10T11:04:05,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,447 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645439) roll requested 2024-12-10T11:04:05,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645447 2024-12-10T11:04:05,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645447 2024-12-10T11:04:05,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645447) roll requested 2024-12-10T11:04:05,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645454 2024-12-10T11:04:05,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645454 2024-12-10T11:04:05,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645454) roll requested 2024-12-10T11:04:05,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645460 2024-12-10T11:04:05,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645460 2024-12-10T11:04:05,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645460) roll requested 2024-12-10T11:04:05,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645467 2024-12-10T11:04:05,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645467 2024-12-10T11:04:05,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645467) roll requested 2024-12-10T11:04:05,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645473 2024-12-10T11:04:05,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645473 2024-12-10T11:04:05,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645473) roll requested 2024-12-10T11:04:05,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645479 2024-12-10T11:04:05,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645479 2024-12-10T11:04:05,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645479) roll requested 2024-12-10T11:04:05,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645486 2024-12-10T11:04:05,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645486 2024-12-10T11:04:05,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645486) roll requested 2024-12-10T11:04:05,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645492 2024-12-10T11:04:05,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645492 2024-12-10T11:04:05,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645492) roll requested 2024-12-10T11:04:05,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645498 2024-12-10T11:04:05,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645498 2024-12-10T11:04:05,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645498) roll requested 2024-12-10T11:04:05,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645504 2024-12-10T11:04:05,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645504 2024-12-10T11:04:05,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645504) roll requested 2024-12-10T11:04:05,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645511 2024-12-10T11:04:05,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645511 2024-12-10T11:04:05,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645511) roll requested 2024-12-10T11:04:05,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645517 2024-12-10T11:04:05,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645517 2024-12-10T11:04:05,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645517) roll requested 2024-12-10T11:04:05,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645523 2024-12-10T11:04:05,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645523 2024-12-10T11:04:05,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645523) roll requested 2024-12-10T11:04:05,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645529 2024-12-10T11:04:05,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645529 2024-12-10T11:04:05,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645529) roll requested 2024-12-10T11:04:05,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645536 2024-12-10T11:04:05,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645536 2024-12-10T11:04:05,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645536) roll requested 2024-12-10T11:04:05,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645544 2024-12-10T11:04:05,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645544 2024-12-10T11:04:05,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645544) roll requested 2024-12-10T11:04:05,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645551 2024-12-10T11:04:05,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645551 2024-12-10T11:04:05,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645551) roll requested 2024-12-10T11:04:05,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645558 2024-12-10T11:04:05,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645558 2024-12-10T11:04:05,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645558) roll requested 2024-12-10T11:04:05,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645566 2024-12-10T11:04:05,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645566 2024-12-10T11:04:05,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645566) roll requested 2024-12-10T11:04:05,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645572 2024-12-10T11:04:05,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645572 2024-12-10T11:04:05,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645572) roll requested 2024-12-10T11:04:05,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645583 2024-12-10T11:04:05,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645583 2024-12-10T11:04:05,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645583) roll requested 2024-12-10T11:04:05,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645593 2024-12-10T11:04:05,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645593 2024-12-10T11:04:05,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645593) roll requested 2024-12-10T11:04:05,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645606 2024-12-10T11:04:05,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645606 2024-12-10T11:04:05,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645606) roll requested 2024-12-10T11:04:05,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645617 2024-12-10T11:04:05,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645617 2024-12-10T11:04:05,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645617) roll requested 2024-12-10T11:04:05,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645626 2024-12-10T11:04:05,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645626 2024-12-10T11:04:05,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645626) roll requested 2024-12-10T11:04:05,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645633 2024-12-10T11:04:05,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645633 2024-12-10T11:04:05,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645633) roll requested 2024-12-10T11:04:05,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645640 2024-12-10T11:04:05,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645640 2024-12-10T11:04:05,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645640) roll requested 2024-12-10T11:04:05,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645648 2024-12-10T11:04:05,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645648 2024-12-10T11:04:05,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645648) roll requested 2024-12-10T11:04:05,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645655 2024-12-10T11:04:05,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645655 2024-12-10T11:04:05,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645655) roll requested 2024-12-10T11:04:05,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645662 2024-12-10T11:04:05,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645662 2024-12-10T11:04:05,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645662) roll requested 2024-12-10T11:04:05,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645668 2024-12-10T11:04:05,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645668 2024-12-10T11:04:05,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645668) roll requested 2024-12-10T11:04:05,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645675 2024-12-10T11:04:05,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645675 2024-12-10T11:04:05,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645675) roll requested 2024-12-10T11:04:05,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645682 2024-12-10T11:04:05,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645682 2024-12-10T11:04:05,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645682) roll requested 2024-12-10T11:04:05,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645689 2024-12-10T11:04:05,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645689 2024-12-10T11:04:05,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645689) roll requested 2024-12-10T11:04:05,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645696 2024-12-10T11:04:05,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645696 2024-12-10T11:04:05,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645696) roll requested 2024-12-10T11:04:05,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645703 2024-12-10T11:04:05,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645703 2024-12-10T11:04:05,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645703) roll requested 2024-12-10T11:04:05,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645710 2024-12-10T11:04:05,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645710 2024-12-10T11:04:05,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645710) roll requested 2024-12-10T11:04:05,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645718 2024-12-10T11:04:05,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645718 2024-12-10T11:04:05,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645718) roll requested 2024-12-10T11:04:05,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645726 2024-12-10T11:04:05,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645726 2024-12-10T11:04:05,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645726) roll requested 2024-12-10T11:04:05,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645734 2024-12-10T11:04:05,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645734 2024-12-10T11:04:05,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645734) roll requested 2024-12-10T11:04:05,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645741 2024-12-10T11:04:05,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645741 2024-12-10T11:04:05,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645741) roll requested 2024-12-10T11:04:05,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645749 2024-12-10T11:04:05,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645749 2024-12-10T11:04:05,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645749) roll requested 2024-12-10T11:04:05,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645756 2024-12-10T11:04:05,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645756 2024-12-10T11:04:05,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645756) roll requested 2024-12-10T11:04:05,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645763 2024-12-10T11:04:05,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645763 2024-12-10T11:04:05,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645763) roll requested 2024-12-10T11:04:05,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645770 2024-12-10T11:04:05,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645770 2024-12-10T11:04:05,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645770) roll requested 2024-12-10T11:04:05,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645777 2024-12-10T11:04:05,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645777 2024-12-10T11:04:05,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645777) roll requested 2024-12-10T11:04:05,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645785 2024-12-10T11:04:05,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645785 2024-12-10T11:04:05,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645785) roll requested 2024-12-10T11:04:05,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645792 2024-12-10T11:04:05,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645792 2024-12-10T11:04:05,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645792) roll requested 2024-12-10T11:04:05,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645799 2024-12-10T11:04:05,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645799 2024-12-10T11:04:05,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645799) roll requested 2024-12-10T11:04:05,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645806 2024-12-10T11:04:05,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645806 2024-12-10T11:04:05,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645806) roll requested 2024-12-10T11:04:05,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645813 2024-12-10T11:04:05,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645813 2024-12-10T11:04:05,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645813) roll requested 2024-12-10T11:04:05,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645820 2024-12-10T11:04:05,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645820 2024-12-10T11:04:05,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645820) roll requested 2024-12-10T11:04:05,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645828 2024-12-10T11:04:05,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645828 2024-12-10T11:04:05,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645828) roll requested 2024-12-10T11:04:05,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645836 2024-12-10T11:04:05,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645836 2024-12-10T11:04:05,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645836) roll requested 2024-12-10T11:04:05,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645846 2024-12-10T11:04:05,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645846 2024-12-10T11:04:05,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645846) roll requested 2024-12-10T11:04:05,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645853 2024-12-10T11:04:05,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645853 2024-12-10T11:04:05,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645853) roll requested 2024-12-10T11:04:05,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645861 2024-12-10T11:04:05,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645861 2024-12-10T11:04:05,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645861) roll requested 2024-12-10T11:04:05,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645868 2024-12-10T11:04:05,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645868 2024-12-10T11:04:05,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645868) roll requested 2024-12-10T11:04:05,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645875 2024-12-10T11:04:05,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645875 2024-12-10T11:04:05,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645875) roll requested 2024-12-10T11:04:05,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645882 2024-12-10T11:04:05,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645882 2024-12-10T11:04:05,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645882) roll requested 2024-12-10T11:04:05,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645890 2024-12-10T11:04:05,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645890 2024-12-10T11:04:05,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645890) roll requested 2024-12-10T11:04:05,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645896 2024-12-10T11:04:05,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645896 2024-12-10T11:04:05,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645896) roll requested 2024-12-10T11:04:05,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645903 2024-12-10T11:04:05,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645903 2024-12-10T11:04:05,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645903) roll requested 2024-12-10T11:04:05,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645910 2024-12-10T11:04:05,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645910 2024-12-10T11:04:05,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645910) roll requested 2024-12-10T11:04:05,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645917 2024-12-10T11:04:05,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645917 2024-12-10T11:04:05,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645917) roll requested 2024-12-10T11:04:05,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645924 2024-12-10T11:04:05,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645924 2024-12-10T11:04:05,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645924) roll requested 2024-12-10T11:04:05,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645931 2024-12-10T11:04:05,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645931 2024-12-10T11:04:05,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645931) roll requested 2024-12-10T11:04:05,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645938 2024-12-10T11:04:05,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645938 2024-12-10T11:04:05,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645938) roll requested 2024-12-10T11:04:05,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645946 2024-12-10T11:04:05,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645946 2024-12-10T11:04:05,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645946) roll requested 2024-12-10T11:04:05,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645952 2024-12-10T11:04:05,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645952 2024-12-10T11:04:05,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645952) roll requested 2024-12-10T11:04:05,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645959 2024-12-10T11:04:05,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645959 2024-12-10T11:04:05,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645959) roll requested 2024-12-10T11:04:05,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645966 2024-12-10T11:04:05,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645966 2024-12-10T11:04:05,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645966) roll requested 2024-12-10T11:04:05,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645973 2024-12-10T11:04:05,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645973 2024-12-10T11:04:05,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645973) roll requested 2024-12-10T11:04:05,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645979 2024-12-10T11:04:05,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645979 2024-12-10T11:04:05,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645979) roll requested 2024-12-10T11:04:05,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645986 2024-12-10T11:04:05,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645986 2024-12-10T11:04:05,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:05,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:05,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:05,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:05,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645986) roll requested 2024-12-10T11:04:05,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828645994 2024-12-10T11:04:05,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:05,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645994 2024-12-10T11:04:06,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828645994) roll requested 2024-12-10T11:04:06,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646000 2024-12-10T11:04:06,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828645994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646000 2024-12-10T11:04:06,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646000) roll requested 2024-12-10T11:04:06,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646006 2024-12-10T11:04:06,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646006 2024-12-10T11:04:06,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646006) roll requested 2024-12-10T11:04:06,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646014 2024-12-10T11:04:06,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646014 2024-12-10T11:04:06,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646014) roll requested 2024-12-10T11:04:06,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646022 2024-12-10T11:04:06,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646022 2024-12-10T11:04:06,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646022) roll requested 2024-12-10T11:04:06,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646029 2024-12-10T11:04:06,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646029 2024-12-10T11:04:06,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646029) roll requested 2024-12-10T11:04:06,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646036 2024-12-10T11:04:06,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646036 2024-12-10T11:04:06,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646036) roll requested 2024-12-10T11:04:06,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646043 2024-12-10T11:04:06,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646043 2024-12-10T11:04:06,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646043) roll requested 2024-12-10T11:04:06,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646050 2024-12-10T11:04:06,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646050 2024-12-10T11:04:06,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646050) roll requested 2024-12-10T11:04:06,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646056 2024-12-10T11:04:06,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646056 2024-12-10T11:04:06,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646056) roll requested 2024-12-10T11:04:06,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646062 2024-12-10T11:04:06,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646062 2024-12-10T11:04:06,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646062) roll requested 2024-12-10T11:04:06,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646069 2024-12-10T11:04:06,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646069 2024-12-10T11:04:06,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646069) roll requested 2024-12-10T11:04:06,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646076 2024-12-10T11:04:06,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646076 2024-12-10T11:04:06,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646076) roll requested 2024-12-10T11:04:06,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646083 2024-12-10T11:04:06,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646083 2024-12-10T11:04:06,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646083) roll requested 2024-12-10T11:04:06,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646089 2024-12-10T11:04:06,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646089 2024-12-10T11:04:06,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646089) roll requested 2024-12-10T11:04:06,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646096 2024-12-10T11:04:06,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646096 2024-12-10T11:04:06,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646096) roll requested 2024-12-10T11:04:06,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646103 2024-12-10T11:04:06,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646103 2024-12-10T11:04:06,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646103) roll requested 2024-12-10T11:04:06,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646110 2024-12-10T11:04:06,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646110 2024-12-10T11:04:06,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646110) roll requested 2024-12-10T11:04:06,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646116 2024-12-10T11:04:06,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646116 2024-12-10T11:04:06,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646116) roll requested 2024-12-10T11:04:06,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646122 2024-12-10T11:04:06,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646122 2024-12-10T11:04:06,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646122) roll requested 2024-12-10T11:04:06,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646129 2024-12-10T11:04:06,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646129 2024-12-10T11:04:06,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646129) roll requested 2024-12-10T11:04:06,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646135 2024-12-10T11:04:06,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646135 2024-12-10T11:04:06,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646135) roll requested 2024-12-10T11:04:06,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646141 2024-12-10T11:04:06,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646141 2024-12-10T11:04:06,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646141) roll requested 2024-12-10T11:04:06,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646148 2024-12-10T11:04:06,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646148 2024-12-10T11:04:06,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646148) roll requested 2024-12-10T11:04:06,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646154 2024-12-10T11:04:06,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646154 2024-12-10T11:04:06,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646154) roll requested 2024-12-10T11:04:06,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646161 2024-12-10T11:04:06,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646161 2024-12-10T11:04:06,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646161) roll requested 2024-12-10T11:04:06,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646168 2024-12-10T11:04:06,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646168 2024-12-10T11:04:06,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646168) roll requested 2024-12-10T11:04:06,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646175 2024-12-10T11:04:06,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646175 2024-12-10T11:04:06,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,182 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,182 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646175) roll requested 2024-12-10T11:04:06,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646182 2024-12-10T11:04:06,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646182 2024-12-10T11:04:06,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646182) roll requested 2024-12-10T11:04:06,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646188 2024-12-10T11:04:06,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646182 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646188 2024-12-10T11:04:06,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646188) roll requested 2024-12-10T11:04:06,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646195 2024-12-10T11:04:06,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646195 2024-12-10T11:04:06,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646195) roll requested 2024-12-10T11:04:06,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646201 2024-12-10T11:04:06,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646201 2024-12-10T11:04:06,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646201) roll requested 2024-12-10T11:04:06,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646208 2024-12-10T11:04:06,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646208 2024-12-10T11:04:06,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646208) roll requested 2024-12-10T11:04:06,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646214 2024-12-10T11:04:06,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646214 2024-12-10T11:04:06,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646214) roll requested 2024-12-10T11:04:06,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646220 2024-12-10T11:04:06,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646220 2024-12-10T11:04:06,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646220) roll requested 2024-12-10T11:04:06,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646229 2024-12-10T11:04:06,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646229 2024-12-10T11:04:06,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646229) roll requested 2024-12-10T11:04:06,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646236 2024-12-10T11:04:06,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646236 2024-12-10T11:04:06,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646236) roll requested 2024-12-10T11:04:06,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646243 2024-12-10T11:04:06,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646243 2024-12-10T11:04:06,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646243) roll requested 2024-12-10T11:04:06,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646249 2024-12-10T11:04:06,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646249 2024-12-10T11:04:06,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646249) roll requested 2024-12-10T11:04:06,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646255 2024-12-10T11:04:06,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646255 2024-12-10T11:04:06,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646255) roll requested 2024-12-10T11:04:06,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646262 2024-12-10T11:04:06,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646262 2024-12-10T11:04:06,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646262) roll requested 2024-12-10T11:04:06,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646268 2024-12-10T11:04:06,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646268 2024-12-10T11:04:06,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646268) roll requested 2024-12-10T11:04:06,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646274 2024-12-10T11:04:06,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646274 2024-12-10T11:04:06,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646274) roll requested 2024-12-10T11:04:06,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646282 2024-12-10T11:04:06,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646282 2024-12-10T11:04:06,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646282) roll requested 2024-12-10T11:04:06,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646289 2024-12-10T11:04:06,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646289 2024-12-10T11:04:06,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,295 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646289) roll requested 2024-12-10T11:04:06,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646296 2024-12-10T11:04:06,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646296 2024-12-10T11:04:06,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646296) roll requested 2024-12-10T11:04:06,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646303 2024-12-10T11:04:06,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646303 2024-12-10T11:04:06,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646303) roll requested 2024-12-10T11:04:06,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646309 2024-12-10T11:04:06,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646309 2024-12-10T11:04:06,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646309) roll requested 2024-12-10T11:04:06,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646316 2024-12-10T11:04:06,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646316 2024-12-10T11:04:06,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646316) roll requested 2024-12-10T11:04:06,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646322 2024-12-10T11:04:06,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646322 2024-12-10T11:04:06,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646322) roll requested 2024-12-10T11:04:06,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646328 2024-12-10T11:04:06,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646328 2024-12-10T11:04:06,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646328) roll requested 2024-12-10T11:04:06,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646335 2024-12-10T11:04:06,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646335 2024-12-10T11:04:06,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646335) roll requested 2024-12-10T11:04:06,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646342 2024-12-10T11:04:06,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646342 2024-12-10T11:04:06,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646342) roll requested 2024-12-10T11:04:06,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646348 2024-12-10T11:04:06,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646348 2024-12-10T11:04:06,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646348) roll requested 2024-12-10T11:04:06,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646355 2024-12-10T11:04:06,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646355 2024-12-10T11:04:06,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646355) roll requested 2024-12-10T11:04:06,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646361 2024-12-10T11:04:06,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646361 2024-12-10T11:04:06,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646361) roll requested 2024-12-10T11:04:06,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646368 2024-12-10T11:04:06,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646368 2024-12-10T11:04:06,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646368) roll requested 2024-12-10T11:04:06,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646374 2024-12-10T11:04:06,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646374 2024-12-10T11:04:06,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646374) roll requested 2024-12-10T11:04:06,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646380 2024-12-10T11:04:06,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646380 2024-12-10T11:04:06,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646380) roll requested 2024-12-10T11:04:06,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646387 2024-12-10T11:04:06,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646387 2024-12-10T11:04:06,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,393 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646387) roll requested 2024-12-10T11:04:06,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646393 2024-12-10T11:04:06,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646393 2024-12-10T11:04:06,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646393) roll requested 2024-12-10T11:04:06,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646399 2024-12-10T11:04:06,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646399 2024-12-10T11:04:06,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646399) roll requested 2024-12-10T11:04:06,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646406 2024-12-10T11:04:06,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646406 2024-12-10T11:04:06,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646406) roll requested 2024-12-10T11:04:06,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646414 2024-12-10T11:04:06,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646414 2024-12-10T11:04:06,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646414) roll requested 2024-12-10T11:04:06,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646426 2024-12-10T11:04:06,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646426 2024-12-10T11:04:06,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646426) roll requested 2024-12-10T11:04:06,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646433 2024-12-10T11:04:06,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646433 2024-12-10T11:04:06,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646433) roll requested 2024-12-10T11:04:06,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646440 2024-12-10T11:04:06,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646440 2024-12-10T11:04:06,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646440) roll requested 2024-12-10T11:04:06,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646448 2024-12-10T11:04:06,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646448 2024-12-10T11:04:06,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646448) roll requested 2024-12-10T11:04:06,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646457 2024-12-10T11:04:06,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646457 2024-12-10T11:04:06,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646457) roll requested 2024-12-10T11:04:06,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646464 2024-12-10T11:04:06,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646464 2024-12-10T11:04:06,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646464) roll requested 2024-12-10T11:04:06,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646473 2024-12-10T11:04:06,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646473 2024-12-10T11:04:06,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646473) roll requested 2024-12-10T11:04:06,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646481 2024-12-10T11:04:06,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646481 2024-12-10T11:04:06,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646481) roll requested 2024-12-10T11:04:06,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646488 2024-12-10T11:04:06,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646488 2024-12-10T11:04:06,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646488) roll requested 2024-12-10T11:04:06,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646497 2024-12-10T11:04:06,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646497 2024-12-10T11:04:06,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646497) roll requested 2024-12-10T11:04:06,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646503 2024-12-10T11:04:06,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646503 2024-12-10T11:04:06,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646503) roll requested 2024-12-10T11:04:06,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646511 2024-12-10T11:04:06,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646511 2024-12-10T11:04:06,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646511) roll requested 2024-12-10T11:04:06,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646518 2024-12-10T11:04:06,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646518 2024-12-10T11:04:06,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646518) roll requested 2024-12-10T11:04:06,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646525 2024-12-10T11:04:06,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646525 2024-12-10T11:04:06,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646525) roll requested 2024-12-10T11:04:06,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646531 2024-12-10T11:04:06,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646531 2024-12-10T11:04:06,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646531) roll requested 2024-12-10T11:04:06,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646538 2024-12-10T11:04:06,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646538 2024-12-10T11:04:06,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646538) roll requested 2024-12-10T11:04:06,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646544 2024-12-10T11:04:06,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646544 2024-12-10T11:04:06,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646544) roll requested 2024-12-10T11:04:06,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646551 2024-12-10T11:04:06,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646551 2024-12-10T11:04:06,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646551) roll requested 2024-12-10T11:04:06,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646558 2024-12-10T11:04:06,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646558 2024-12-10T11:04:06,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646558) roll requested 2024-12-10T11:04:06,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646565 2024-12-10T11:04:06,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646565 2024-12-10T11:04:06,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646565) roll requested 2024-12-10T11:04:06,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646573 2024-12-10T11:04:06,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646573 2024-12-10T11:04:06,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646573) roll requested 2024-12-10T11:04:06,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646579 2024-12-10T11:04:06,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646579 2024-12-10T11:04:06,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646579) roll requested 2024-12-10T11:04:06,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646587 2024-12-10T11:04:06,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646587 2024-12-10T11:04:06,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646587) roll requested 2024-12-10T11:04:06,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646593 2024-12-10T11:04:06,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646593 2024-12-10T11:04:06,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646593) roll requested 2024-12-10T11:04:06,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646600 2024-12-10T11:04:06,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646600 2024-12-10T11:04:06,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,606 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,606 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,606 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646600) roll requested 2024-12-10T11:04:06,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646606 2024-12-10T11:04:06,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646606 2024-12-10T11:04:06,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646606) roll requested 2024-12-10T11:04:06,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646613 2024-12-10T11:04:06,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646613 2024-12-10T11:04:06,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646613) roll requested 2024-12-10T11:04:06,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646623 2024-12-10T11:04:06,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646623 2024-12-10T11:04:06,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646623) roll requested 2024-12-10T11:04:06,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646631 2024-12-10T11:04:06,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646631 2024-12-10T11:04:06,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646631) roll requested 2024-12-10T11:04:06,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646637 2024-12-10T11:04:06,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646637 2024-12-10T11:04:06,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646637) roll requested 2024-12-10T11:04:06,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646647 2024-12-10T11:04:06,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646647 2024-12-10T11:04:06,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646647) roll requested 2024-12-10T11:04:06,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646658 2024-12-10T11:04:06,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646658 2024-12-10T11:04:06,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646658) roll requested 2024-12-10T11:04:06,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646674 2024-12-10T11:04:06,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646674 2024-12-10T11:04:06,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646674) roll requested 2024-12-10T11:04:06,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646687 2024-12-10T11:04:06,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646687 2024-12-10T11:04:06,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646687) roll requested 2024-12-10T11:04:06,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646694 2024-12-10T11:04:06,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646694 2024-12-10T11:04:06,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646694) roll requested 2024-12-10T11:04:06,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646708 2024-12-10T11:04:06,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646708 2024-12-10T11:04:06,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646708) roll requested 2024-12-10T11:04:06,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646717 2024-12-10T11:04:06,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646717 2024-12-10T11:04:06,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646717) roll requested 2024-12-10T11:04:06,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646726 2024-12-10T11:04:06,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646726 2024-12-10T11:04:06,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646726) roll requested 2024-12-10T11:04:06,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646735 2024-12-10T11:04:06,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646735 2024-12-10T11:04:06,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646735) roll requested 2024-12-10T11:04:06,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646746 2024-12-10T11:04:06,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646746 2024-12-10T11:04:06,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646746) roll requested 2024-12-10T11:04:06,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646753 2024-12-10T11:04:06,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646753 2024-12-10T11:04:06,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646753) roll requested 2024-12-10T11:04:06,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646760 2024-12-10T11:04:06,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646760 2024-12-10T11:04:06,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646760) roll requested 2024-12-10T11:04:06,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646767 2024-12-10T11:04:06,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646767 2024-12-10T11:04:06,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646767) roll requested 2024-12-10T11:04:06,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646786 2024-12-10T11:04:06,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646786 2024-12-10T11:04:06,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646786) roll requested 2024-12-10T11:04:06,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646794 2024-12-10T11:04:06,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646794 2024-12-10T11:04:06,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646794) roll requested 2024-12-10T11:04:06,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646801 2024-12-10T11:04:06,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646801 2024-12-10T11:04:06,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646801) roll requested 2024-12-10T11:04:06,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646810 2024-12-10T11:04:06,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646810 2024-12-10T11:04:06,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646810) roll requested 2024-12-10T11:04:06,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646817 2024-12-10T11:04:06,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646817 2024-12-10T11:04:06,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646817) roll requested 2024-12-10T11:04:06,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646823 2024-12-10T11:04:06,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646823 2024-12-10T11:04:06,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646823) roll requested 2024-12-10T11:04:06,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646830 2024-12-10T11:04:06,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646830 2024-12-10T11:04:06,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646830) roll requested 2024-12-10T11:04:06,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646837 2024-12-10T11:04:06,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646837 2024-12-10T11:04:06,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646837) roll requested 2024-12-10T11:04:06,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646844 2024-12-10T11:04:06,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646844 2024-12-10T11:04:06,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646844) roll requested 2024-12-10T11:04:06,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646855 2024-12-10T11:04:06,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646855 2024-12-10T11:04:06,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646855) roll requested 2024-12-10T11:04:06,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646866 2024-12-10T11:04:06,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646855 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646866 2024-12-10T11:04:06,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646866) roll requested 2024-12-10T11:04:06,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646875 2024-12-10T11:04:06,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646875 2024-12-10T11:04:06,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646875) roll requested 2024-12-10T11:04:06,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646886 2024-12-10T11:04:06,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646886 2024-12-10T11:04:06,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646886) roll requested 2024-12-10T11:04:06,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646894 2024-12-10T11:04:06,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646894 2024-12-10T11:04:06,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646894) roll requested 2024-12-10T11:04:06,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646901 2024-12-10T11:04:06,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646901 2024-12-10T11:04:06,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646901) roll requested 2024-12-10T11:04:06,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646907 2024-12-10T11:04:06,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646907 2024-12-10T11:04:06,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646907) roll requested 2024-12-10T11:04:06,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646914 2024-12-10T11:04:06,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646914 2024-12-10T11:04:06,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646914) roll requested 2024-12-10T11:04:06,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646921 2024-12-10T11:04:06,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646921 2024-12-10T11:04:06,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646921) roll requested 2024-12-10T11:04:06,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646927 2024-12-10T11:04:06,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646927 2024-12-10T11:04:06,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646927) roll requested 2024-12-10T11:04:06,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646934 2024-12-10T11:04:06,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646934 2024-12-10T11:04:06,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646934) roll requested 2024-12-10T11:04:06,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646940 2024-12-10T11:04:06,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646940 2024-12-10T11:04:06,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646940) roll requested 2024-12-10T11:04:06,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646947 2024-12-10T11:04:06,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646947 2024-12-10T11:04:06,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646947) roll requested 2024-12-10T11:04:06,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646953 2024-12-10T11:04:06,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646953 2024-12-10T11:04:06,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646953) roll requested 2024-12-10T11:04:06,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646959 2024-12-10T11:04:06,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646959 2024-12-10T11:04:06,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646959) roll requested 2024-12-10T11:04:06,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646966 2024-12-10T11:04:06,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646966 2024-12-10T11:04:06,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646966) roll requested 2024-12-10T11:04:06,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646972 2024-12-10T11:04:06,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646972 2024-12-10T11:04:06,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646972) roll requested 2024-12-10T11:04:06,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646979 2024-12-10T11:04:06,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646979 2024-12-10T11:04:06,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646979) roll requested 2024-12-10T11:04:06,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646986 2024-12-10T11:04:06,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646986 2024-12-10T11:04:06,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646986) roll requested 2024-12-10T11:04:06,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646992 2024-12-10T11:04:06,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:06,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646992 2024-12-10T11:04:06,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:06,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:06,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:06,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:06,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646992) roll requested 2024-12-10T11:04:06,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828646998 2024-12-10T11:04:07,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646998 2024-12-10T11:04:07,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828646998) roll requested 2024-12-10T11:04:07,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647005 2024-12-10T11:04:07,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828646998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647005 2024-12-10T11:04:07,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647005) roll requested 2024-12-10T11:04:07,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647013 2024-12-10T11:04:07,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647013 2024-12-10T11:04:07,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647013) roll requested 2024-12-10T11:04:07,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647019 2024-12-10T11:04:07,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647019 2024-12-10T11:04:07,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647019) roll requested 2024-12-10T11:04:07,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647026 2024-12-10T11:04:07,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647026 2024-12-10T11:04:07,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647026) roll requested 2024-12-10T11:04:07,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647032 2024-12-10T11:04:07,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647032 2024-12-10T11:04:07,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,039 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647032) roll requested 2024-12-10T11:04:07,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647039 2024-12-10T11:04:07,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647039 2024-12-10T11:04:07,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647039) roll requested 2024-12-10T11:04:07,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647045 2024-12-10T11:04:07,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647045 2024-12-10T11:04:07,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647045) roll requested 2024-12-10T11:04:07,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647051 2024-12-10T11:04:07,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647051 2024-12-10T11:04:07,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647051) roll requested 2024-12-10T11:04:07,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647058 2024-12-10T11:04:07,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647058 2024-12-10T11:04:07,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647058) roll requested 2024-12-10T11:04:07,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647064 2024-12-10T11:04:07,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647064 2024-12-10T11:04:07,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647064) roll requested 2024-12-10T11:04:07,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647070 2024-12-10T11:04:07,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647070 2024-12-10T11:04:07,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647070) roll requested 2024-12-10T11:04:07,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647076 2024-12-10T11:04:07,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647076 2024-12-10T11:04:07,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647076) roll requested 2024-12-10T11:04:07,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647087 2024-12-10T11:04:07,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647087 2024-12-10T11:04:07,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647087) roll requested 2024-12-10T11:04:07,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647093 2024-12-10T11:04:07,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647093 2024-12-10T11:04:07,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647093) roll requested 2024-12-10T11:04:07,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647101 2024-12-10T11:04:07,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647101 2024-12-10T11:04:07,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647101) roll requested 2024-12-10T11:04:07,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647107 2024-12-10T11:04:07,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647107 2024-12-10T11:04:07,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647107) roll requested 2024-12-10T11:04:07,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647114 2024-12-10T11:04:07,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647114 2024-12-10T11:04:07,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647114) roll requested 2024-12-10T11:04:07,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647120 2024-12-10T11:04:07,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647120 2024-12-10T11:04:07,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647120) roll requested 2024-12-10T11:04:07,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647126 2024-12-10T11:04:07,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647126 2024-12-10T11:04:07,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647126) roll requested 2024-12-10T11:04:07,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647132 2024-12-10T11:04:07,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647132 2024-12-10T11:04:07,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647132) roll requested 2024-12-10T11:04:07,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647139 2024-12-10T11:04:07,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647139 2024-12-10T11:04:07,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647139) roll requested 2024-12-10T11:04:07,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647146 2024-12-10T11:04:07,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647146 2024-12-10T11:04:07,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647146) roll requested 2024-12-10T11:04:07,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647153 2024-12-10T11:04:07,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647153 2024-12-10T11:04:07,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647153) roll requested 2024-12-10T11:04:07,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647159 2024-12-10T11:04:07,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647159 2024-12-10T11:04:07,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647159) roll requested 2024-12-10T11:04:07,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647166 2024-12-10T11:04:07,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647166 2024-12-10T11:04:07,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647166) roll requested 2024-12-10T11:04:07,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647172 2024-12-10T11:04:07,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647172 2024-12-10T11:04:07,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647172) roll requested 2024-12-10T11:04:07,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647179 2024-12-10T11:04:07,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647179 2024-12-10T11:04:07,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647179) roll requested 2024-12-10T11:04:07,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647185 2024-12-10T11:04:07,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647185 2024-12-10T11:04:07,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647185) roll requested 2024-12-10T11:04:07,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647192 2024-12-10T11:04:07,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647192 2024-12-10T11:04:07,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647192) roll requested 2024-12-10T11:04:07,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647198 2024-12-10T11:04:07,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647198 2024-12-10T11:04:07,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647198) roll requested 2024-12-10T11:04:07,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647205 2024-12-10T11:04:07,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647205 2024-12-10T11:04:07,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647205) roll requested 2024-12-10T11:04:07,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647212 2024-12-10T11:04:07,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647212 2024-12-10T11:04:07,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647212) roll requested 2024-12-10T11:04:07,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647218 2024-12-10T11:04:07,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647218 2024-12-10T11:04:07,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647218) roll requested 2024-12-10T11:04:07,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647224 2024-12-10T11:04:07,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647224 2024-12-10T11:04:07,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647224) roll requested 2024-12-10T11:04:07,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647231 2024-12-10T11:04:07,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647231 2024-12-10T11:04:07,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647231) roll requested 2024-12-10T11:04:07,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647237 2024-12-10T11:04:07,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647237 2024-12-10T11:04:07,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647237) roll requested 2024-12-10T11:04:07,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647245 2024-12-10T11:04:07,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647245 2024-12-10T11:04:07,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647245) roll requested 2024-12-10T11:04:07,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647251 2024-12-10T11:04:07,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647251 2024-12-10T11:04:07,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647251) roll requested 2024-12-10T11:04:07,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647257 2024-12-10T11:04:07,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647257 2024-12-10T11:04:07,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647257) roll requested 2024-12-10T11:04:07,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647264 2024-12-10T11:04:07,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647264 2024-12-10T11:04:07,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647264) roll requested 2024-12-10T11:04:07,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647270 2024-12-10T11:04:07,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647270 2024-12-10T11:04:07,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647270) roll requested 2024-12-10T11:04:07,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647277 2024-12-10T11:04:07,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647277 2024-12-10T11:04:07,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647277) roll requested 2024-12-10T11:04:07,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647283 2024-12-10T11:04:07,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647283 2024-12-10T11:04:07,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647283) roll requested 2024-12-10T11:04:07,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647290 2024-12-10T11:04:07,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647290 2024-12-10T11:04:07,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,298 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647290) roll requested 2024-12-10T11:04:07,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647298 2024-12-10T11:04:07,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647298 2024-12-10T11:04:07,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647298) roll requested 2024-12-10T11:04:07,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647304 2024-12-10T11:04:07,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647298 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647304 2024-12-10T11:04:07,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647304) roll requested 2024-12-10T11:04:07,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647320 2024-12-10T11:04:07,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647320 2024-12-10T11:04:07,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647320) roll requested 2024-12-10T11:04:07,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647328 2024-12-10T11:04:07,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647328 2024-12-10T11:04:07,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647328) roll requested 2024-12-10T11:04:07,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647337 2024-12-10T11:04:07,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647337 2024-12-10T11:04:07,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647337) roll requested 2024-12-10T11:04:07,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647352 2024-12-10T11:04:07,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647352 2024-12-10T11:04:07,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647352) roll requested 2024-12-10T11:04:07,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647361 2024-12-10T11:04:07,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647361 2024-12-10T11:04:07,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647361) roll requested 2024-12-10T11:04:07,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647374 2024-12-10T11:04:07,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647374 2024-12-10T11:04:07,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647374) roll requested 2024-12-10T11:04:07,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647385 2024-12-10T11:04:07,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647385 2024-12-10T11:04:07,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647385) roll requested 2024-12-10T11:04:07,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647394 2024-12-10T11:04:07,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647394 2024-12-10T11:04:07,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647394) roll requested 2024-12-10T11:04:07,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647406 2024-12-10T11:04:07,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647406 2024-12-10T11:04:07,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647406) roll requested 2024-12-10T11:04:07,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647415 2024-12-10T11:04:07,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647415 2024-12-10T11:04:07,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647415) roll requested 2024-12-10T11:04:07,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647429 2024-12-10T11:04:07,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647429 2024-12-10T11:04:07,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647429) roll requested 2024-12-10T11:04:07,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647443 2024-12-10T11:04:07,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647443 2024-12-10T11:04:07,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647443) roll requested 2024-12-10T11:04:07,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647450 2024-12-10T11:04:07,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647450 2024-12-10T11:04:07,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647450) roll requested 2024-12-10T11:04:07,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647459 2024-12-10T11:04:07,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647459 2024-12-10T11:04:07,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647459) roll requested 2024-12-10T11:04:07,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647466 2024-12-10T11:04:07,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647466 2024-12-10T11:04:07,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647466) roll requested 2024-12-10T11:04:07,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647475 2024-12-10T11:04:07,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647475 2024-12-10T11:04:07,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647475) roll requested 2024-12-10T11:04:07,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647484 2024-12-10T11:04:07,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647484 2024-12-10T11:04:07,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647484) roll requested 2024-12-10T11:04:07,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647492 2024-12-10T11:04:07,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647492 2024-12-10T11:04:07,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647492) roll requested 2024-12-10T11:04:07,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647498 2024-12-10T11:04:07,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647498 2024-12-10T11:04:07,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647498) roll requested 2024-12-10T11:04:07,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647505 2024-12-10T11:04:07,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647505 2024-12-10T11:04:07,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647505) roll requested 2024-12-10T11:04:07,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647511 2024-12-10T11:04:07,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647511 2024-12-10T11:04:07,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647511) roll requested 2024-12-10T11:04:07,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647518 2024-12-10T11:04:07,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647518 2024-12-10T11:04:07,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647518) roll requested 2024-12-10T11:04:07,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647524 2024-12-10T11:04:07,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647524 2024-12-10T11:04:07,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647524) roll requested 2024-12-10T11:04:07,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647530 2024-12-10T11:04:07,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647530 2024-12-10T11:04:07,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647530) roll requested 2024-12-10T11:04:07,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647537 2024-12-10T11:04:07,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647537 2024-12-10T11:04:07,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647537) roll requested 2024-12-10T11:04:07,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647543 2024-12-10T11:04:07,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647543 2024-12-10T11:04:07,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647543) roll requested 2024-12-10T11:04:07,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647550 2024-12-10T11:04:07,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647550 2024-12-10T11:04:07,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647550) roll requested 2024-12-10T11:04:07,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647556 2024-12-10T11:04:07,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647556 2024-12-10T11:04:07,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647556) roll requested 2024-12-10T11:04:07,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647563 2024-12-10T11:04:07,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647563 2024-12-10T11:04:07,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647563) roll requested 2024-12-10T11:04:07,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647569 2024-12-10T11:04:07,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647569 2024-12-10T11:04:07,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647569) roll requested 2024-12-10T11:04:07,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647575 2024-12-10T11:04:07,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647575 2024-12-10T11:04:07,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647575) roll requested 2024-12-10T11:04:07,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647582 2024-12-10T11:04:07,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647582 2024-12-10T11:04:07,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647582) roll requested 2024-12-10T11:04:07,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647588 2024-12-10T11:04:07,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647588 2024-12-10T11:04:07,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647588) roll requested 2024-12-10T11:04:07,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647595 2024-12-10T11:04:07,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647595 2024-12-10T11:04:07,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647595) roll requested 2024-12-10T11:04:07,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647601 2024-12-10T11:04:07,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647601 2024-12-10T11:04:07,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647601) roll requested 2024-12-10T11:04:07,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647610 2024-12-10T11:04:07,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647610 2024-12-10T11:04:07,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647610) roll requested 2024-12-10T11:04:07,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647619 2024-12-10T11:04:07,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647619 2024-12-10T11:04:07,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647619) roll requested 2024-12-10T11:04:07,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647626 2024-12-10T11:04:07,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647626 2024-12-10T11:04:07,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647626) roll requested 2024-12-10T11:04:07,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647636 2024-12-10T11:04:07,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647636 2024-12-10T11:04:07,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,643 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647636) roll requested 2024-12-10T11:04:07,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647643 2024-12-10T11:04:07,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647643 2024-12-10T11:04:07,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647643) roll requested 2024-12-10T11:04:07,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647652 2024-12-10T11:04:07,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647652 2024-12-10T11:04:07,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647652) roll requested 2024-12-10T11:04:07,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647659 2024-12-10T11:04:07,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647659 2024-12-10T11:04:07,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647659) roll requested 2024-12-10T11:04:07,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647666 2024-12-10T11:04:07,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647666 2024-12-10T11:04:07,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647666) roll requested 2024-12-10T11:04:07,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647674 2024-12-10T11:04:07,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647674 2024-12-10T11:04:07,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647674) roll requested 2024-12-10T11:04:07,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647681 2024-12-10T11:04:07,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647681 2024-12-10T11:04:07,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647681) roll requested 2024-12-10T11:04:07,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647689 2024-12-10T11:04:07,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647689 2024-12-10T11:04:07,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,698 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647689) roll requested 2024-12-10T11:04:07,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647698 2024-12-10T11:04:07,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647698 2024-12-10T11:04:07,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647698) roll requested 2024-12-10T11:04:07,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647704 2024-12-10T11:04:07,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647704 2024-12-10T11:04:07,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647704) roll requested 2024-12-10T11:04:07,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647714 2024-12-10T11:04:07,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647714 2024-12-10T11:04:07,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647714) roll requested 2024-12-10T11:04:07,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647720 2024-12-10T11:04:07,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647720 2024-12-10T11:04:07,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647720) roll requested 2024-12-10T11:04:07,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647730 2024-12-10T11:04:07,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647730 2024-12-10T11:04:07,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647730) roll requested 2024-12-10T11:04:07,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647737 2024-12-10T11:04:07,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647737 2024-12-10T11:04:07,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647737) roll requested 2024-12-10T11:04:07,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647744 2024-12-10T11:04:07,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647744 2024-12-10T11:04:07,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647744) roll requested 2024-12-10T11:04:07,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647753 2024-12-10T11:04:07,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647753 2024-12-10T11:04:07,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647753) roll requested 2024-12-10T11:04:07,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647760 2024-12-10T11:04:07,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647760 2024-12-10T11:04:07,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647760) roll requested 2024-12-10T11:04:07,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647768 2024-12-10T11:04:07,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647768 2024-12-10T11:04:07,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647768) roll requested 2024-12-10T11:04:07,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647788 2024-12-10T11:04:07,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647788 2024-12-10T11:04:07,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647788) roll requested 2024-12-10T11:04:07,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647796 2024-12-10T11:04:07,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647796 2024-12-10T11:04:07,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647796) roll requested 2024-12-10T11:04:07,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647803 2024-12-10T11:04:07,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647803 2024-12-10T11:04:07,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647803) roll requested 2024-12-10T11:04:07,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647813 2024-12-10T11:04:07,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647813 2024-12-10T11:04:07,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647813) roll requested 2024-12-10T11:04:07,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647821 2024-12-10T11:04:07,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647821 2024-12-10T11:04:07,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647821) roll requested 2024-12-10T11:04:07,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647830 2024-12-10T11:04:07,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647830 2024-12-10T11:04:07,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647830) roll requested 2024-12-10T11:04:07,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647842 2024-12-10T11:04:07,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647842 2024-12-10T11:04:07,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647842) roll requested 2024-12-10T11:04:07,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647849 2024-12-10T11:04:07,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647849 2024-12-10T11:04:07,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647849) roll requested 2024-12-10T11:04:07,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647858 2024-12-10T11:04:07,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647858 2024-12-10T11:04:07,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647858) roll requested 2024-12-10T11:04:07,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647868 2024-12-10T11:04:07,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647868 2024-12-10T11:04:07,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647868) roll requested 2024-12-10T11:04:07,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647881 2024-12-10T11:04:07,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647881 2024-12-10T11:04:07,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647881) roll requested 2024-12-10T11:04:07,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647890 2024-12-10T11:04:07,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647890 2024-12-10T11:04:07,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647890) roll requested 2024-12-10T11:04:07,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647900 2024-12-10T11:04:07,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647900 2024-12-10T11:04:07,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647900) roll requested 2024-12-10T11:04:07,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647908 2024-12-10T11:04:07,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647908 2024-12-10T11:04:07,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647908) roll requested 2024-12-10T11:04:07,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647915 2024-12-10T11:04:07,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647915 2024-12-10T11:04:07,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647915) roll requested 2024-12-10T11:04:07,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647926 2024-12-10T11:04:07,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647926 2024-12-10T11:04:07,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647926) roll requested 2024-12-10T11:04:07,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647933 2024-12-10T11:04:07,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647933 2024-12-10T11:04:07,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647933) roll requested 2024-12-10T11:04:07,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647940 2024-12-10T11:04:07,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647940 2024-12-10T11:04:07,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647940) roll requested 2024-12-10T11:04:07,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647949 2024-12-10T11:04:07,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647949 2024-12-10T11:04:07,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647949) roll requested 2024-12-10T11:04:07,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647956 2024-12-10T11:04:07,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647956 2024-12-10T11:04:07,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647956) roll requested 2024-12-10T11:04:07,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647962 2024-12-10T11:04:07,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647962 2024-12-10T11:04:07,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647962) roll requested 2024-12-10T11:04:07,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647970 2024-12-10T11:04:07,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647970 2024-12-10T11:04:07,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647970) roll requested 2024-12-10T11:04:07,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647977 2024-12-10T11:04:07,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647977 2024-12-10T11:04:07,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647977) roll requested 2024-12-10T11:04:07,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647986 2024-12-10T11:04:07,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:07,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647986 2024-12-10T11:04:07,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:07,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:07,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:07,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:07,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647986) roll requested 2024-12-10T11:04:07,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828647996 2024-12-10T11:04:08,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647996 2024-12-10T11:04:08,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828647996) roll requested 2024-12-10T11:04:08,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648004 2024-12-10T11:04:08,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828647996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648004 2024-12-10T11:04:08,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648004) roll requested 2024-12-10T11:04:08,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648013 2024-12-10T11:04:08,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648013 2024-12-10T11:04:08,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648013) roll requested 2024-12-10T11:04:08,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648020 2024-12-10T11:04:08,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648020 2024-12-10T11:04:08,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648020) roll requested 2024-12-10T11:04:08,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648028 2024-12-10T11:04:08,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648028 2024-12-10T11:04:08,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648028) roll requested 2024-12-10T11:04:08,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648035 2024-12-10T11:04:08,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648035 2024-12-10T11:04:08,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648035) roll requested 2024-12-10T11:04:08,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648041 2024-12-10T11:04:08,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648041 2024-12-10T11:04:08,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648041) roll requested 2024-12-10T11:04:08,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648049 2024-12-10T11:04:08,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648049 2024-12-10T11:04:08,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648049) roll requested 2024-12-10T11:04:08,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648057 2024-12-10T11:04:08,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648057 2024-12-10T11:04:08,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648057) roll requested 2024-12-10T11:04:08,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648063 2024-12-10T11:04:08,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648063 2024-12-10T11:04:08,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648063) roll requested 2024-12-10T11:04:08,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648070 2024-12-10T11:04:08,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648070 2024-12-10T11:04:08,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648070) roll requested 2024-12-10T11:04:08,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648079 2024-12-10T11:04:08,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648079 2024-12-10T11:04:08,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648079) roll requested 2024-12-10T11:04:08,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648086 2024-12-10T11:04:08,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648086 2024-12-10T11:04:08,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648086) roll requested 2024-12-10T11:04:08,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648092 2024-12-10T11:04:08,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648092 2024-12-10T11:04:08,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648092) roll requested 2024-12-10T11:04:08,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648104 2024-12-10T11:04:08,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648104 2024-12-10T11:04:08,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648104) roll requested 2024-12-10T11:04:08,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648111 2024-12-10T11:04:08,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648111 2024-12-10T11:04:08,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648111) roll requested 2024-12-10T11:04:08,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648120 2024-12-10T11:04:08,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648120 2024-12-10T11:04:08,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648120) roll requested 2024-12-10T11:04:08,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648130 2024-12-10T11:04:08,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648130 2024-12-10T11:04:08,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648130) roll requested 2024-12-10T11:04:08,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648136 2024-12-10T11:04:08,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648136 2024-12-10T11:04:08,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648136) roll requested 2024-12-10T11:04:08,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648144 2024-12-10T11:04:08,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648144 2024-12-10T11:04:08,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648144) roll requested 2024-12-10T11:04:08,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648154 2024-12-10T11:04:08,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648154 2024-12-10T11:04:08,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648154) roll requested 2024-12-10T11:04:08,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648162 2024-12-10T11:04:08,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648162 2024-12-10T11:04:08,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648162) roll requested 2024-12-10T11:04:08,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648170 2024-12-10T11:04:08,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648170 2024-12-10T11:04:08,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648170) roll requested 2024-12-10T11:04:08,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648178 2024-12-10T11:04:08,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648178 2024-12-10T11:04:08,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648178) roll requested 2024-12-10T11:04:08,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648186 2024-12-10T11:04:08,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648186 2024-12-10T11:04:08,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648186) roll requested 2024-12-10T11:04:08,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648194 2024-12-10T11:04:08,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648194 2024-12-10T11:04:08,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648194) roll requested 2024-12-10T11:04:08,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648202 2024-12-10T11:04:08,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648202 2024-12-10T11:04:08,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648202) roll requested 2024-12-10T11:04:08,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648210 2024-12-10T11:04:08,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648210 2024-12-10T11:04:08,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648210) roll requested 2024-12-10T11:04:08,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648216 2024-12-10T11:04:08,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648216 2024-12-10T11:04:08,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648216) roll requested 2024-12-10T11:04:08,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648233 2024-12-10T11:04:08,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648233 2024-12-10T11:04:08,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648233) roll requested 2024-12-10T11:04:08,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648242 2024-12-10T11:04:08,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648242 2024-12-10T11:04:08,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648242) roll requested 2024-12-10T11:04:08,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648254 2024-12-10T11:04:08,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648254 2024-12-10T11:04:08,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648254) roll requested 2024-12-10T11:04:08,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648262 2024-12-10T11:04:08,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648262 2024-12-10T11:04:08,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648262) roll requested 2024-12-10T11:04:08,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648271 2024-12-10T11:04:08,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648271 2024-12-10T11:04:08,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648271) roll requested 2024-12-10T11:04:08,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648280 2024-12-10T11:04:08,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648280 2024-12-10T11:04:08,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648280) roll requested 2024-12-10T11:04:08,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648288 2024-12-10T11:04:08,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648288 2024-12-10T11:04:08,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,295 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648288) roll requested 2024-12-10T11:04:08,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648295 2024-12-10T11:04:08,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648295 2024-12-10T11:04:08,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648295) roll requested 2024-12-10T11:04:08,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648303 2024-12-10T11:04:08,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648303 2024-12-10T11:04:08,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648303) roll requested 2024-12-10T11:04:08,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648323 2024-12-10T11:04:08,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648323 2024-12-10T11:04:08,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648323) roll requested 2024-12-10T11:04:08,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648334 2024-12-10T11:04:08,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648334 2024-12-10T11:04:08,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648334) roll requested 2024-12-10T11:04:08,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648341 2024-12-10T11:04:08,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648341 2024-12-10T11:04:08,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648341) roll requested 2024-12-10T11:04:08,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648349 2024-12-10T11:04:08,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648349 2024-12-10T11:04:08,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648349) roll requested 2024-12-10T11:04:08,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648362 2024-12-10T11:04:08,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648362 2024-12-10T11:04:08,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648362) roll requested 2024-12-10T11:04:08,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648370 2024-12-10T11:04:08,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648370 2024-12-10T11:04:08,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648370) roll requested 2024-12-10T11:04:08,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648378 2024-12-10T11:04:08,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648378 2024-12-10T11:04:08,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648378) roll requested 2024-12-10T11:04:08,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648385 2024-12-10T11:04:08,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648385 2024-12-10T11:04:08,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648385) roll requested 2024-12-10T11:04:08,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648397 2024-12-10T11:04:08,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648397 2024-12-10T11:04:08,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648397) roll requested 2024-12-10T11:04:08,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648407 2024-12-10T11:04:08,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648407 2024-12-10T11:04:08,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648407) roll requested 2024-12-10T11:04:08,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648429 2024-12-10T11:04:08,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648429 2024-12-10T11:04:08,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648429) roll requested 2024-12-10T11:04:08,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648443 2024-12-10T11:04:08,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648443 2024-12-10T11:04:08,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648443) roll requested 2024-12-10T11:04:08,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648451 2024-12-10T11:04:08,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648451 2024-12-10T11:04:08,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648451) roll requested 2024-12-10T11:04:08,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648459 2024-12-10T11:04:08,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648459 2024-12-10T11:04:08,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648459) roll requested 2024-12-10T11:04:08,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648469 2024-12-10T11:04:08,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648469 2024-12-10T11:04:08,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648469) roll requested 2024-12-10T11:04:08,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648477 2024-12-10T11:04:08,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648477 2024-12-10T11:04:08,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648477) roll requested 2024-12-10T11:04:08,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648485 2024-12-10T11:04:08,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648485 2024-12-10T11:04:08,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648485) roll requested 2024-12-10T11:04:08,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648493 2024-12-10T11:04:08,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648493 2024-12-10T11:04:08,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648493) roll requested 2024-12-10T11:04:08,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648506 2024-12-10T11:04:08,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648506 2024-12-10T11:04:08,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648506) roll requested 2024-12-10T11:04:08,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648522 2024-12-10T11:04:08,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648522 2024-12-10T11:04:08,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648522) roll requested 2024-12-10T11:04:08,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648530 2024-12-10T11:04:08,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648530 2024-12-10T11:04:08,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648530) roll requested 2024-12-10T11:04:08,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648541 2024-12-10T11:04:08,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648541 2024-12-10T11:04:08,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648541) roll requested 2024-12-10T11:04:08,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648551 2024-12-10T11:04:08,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648551 2024-12-10T11:04:08,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648551) roll requested 2024-12-10T11:04:08,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648562 2024-12-10T11:04:08,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648562 2024-12-10T11:04:08,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648562) roll requested 2024-12-10T11:04:08,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648578 2024-12-10T11:04:08,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648578 2024-12-10T11:04:08,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648578) roll requested 2024-12-10T11:04:08,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648592 2024-12-10T11:04:08,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648592 2024-12-10T11:04:08,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648592) roll requested 2024-12-10T11:04:08,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648605 2024-12-10T11:04:08,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648605 2024-12-10T11:04:08,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648605) roll requested 2024-12-10T11:04:08,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648618 2024-12-10T11:04:08,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648618 2024-12-10T11:04:08,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648618) roll requested 2024-12-10T11:04:08,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648634 2024-12-10T11:04:08,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648634 2024-12-10T11:04:08,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648634) roll requested 2024-12-10T11:04:08,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648654 2024-12-10T11:04:08,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648654 2024-12-10T11:04:08,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648654) roll requested 2024-12-10T11:04:08,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648670 2024-12-10T11:04:08,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648670 2024-12-10T11:04:08,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,698 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648670) roll requested 2024-12-10T11:04:08,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648698 2024-12-10T11:04:08,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648698 2024-12-10T11:04:08,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648698) roll requested 2024-12-10T11:04:08,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648720 2024-12-10T11:04:08,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648720 2024-12-10T11:04:08,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648720) roll requested 2024-12-10T11:04:08,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648728 2024-12-10T11:04:08,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648728 2024-12-10T11:04:08,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648728) roll requested 2024-12-10T11:04:08,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648738 2024-12-10T11:04:08,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648738 2024-12-10T11:04:08,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648738) roll requested 2024-12-10T11:04:08,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648748 2024-12-10T11:04:08,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648748 2024-12-10T11:04:08,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648748) roll requested 2024-12-10T11:04:08,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648758 2024-12-10T11:04:08,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648758 2024-12-10T11:04:08,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648758) roll requested 2024-12-10T11:04:08,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648769 2024-12-10T11:04:08,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648769 2024-12-10T11:04:08,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648769) roll requested 2024-12-10T11:04:08,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648779 2024-12-10T11:04:08,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648779 2024-12-10T11:04:08,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648779) roll requested 2024-12-10T11:04:08,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648794 2024-12-10T11:04:08,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648794 2024-12-10T11:04:08,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648794) roll requested 2024-12-10T11:04:08,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648807 2024-12-10T11:04:08,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648807 2024-12-10T11:04:08,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648807) roll requested 2024-12-10T11:04:08,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648818 2024-12-10T11:04:08,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648818 2024-12-10T11:04:08,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648818) roll requested 2024-12-10T11:04:08,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648825 2024-12-10T11:04:08,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648825 2024-12-10T11:04:08,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,832 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648825) roll requested 2024-12-10T11:04:08,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648832 2024-12-10T11:04:08,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648832 2024-12-10T11:04:08,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648832) roll requested 2024-12-10T11:04:08,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648841 2024-12-10T11:04:08,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648841 2024-12-10T11:04:08,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648841) roll requested 2024-12-10T11:04:08,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648849 2024-12-10T11:04:08,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648849 2024-12-10T11:04:08,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648849) roll requested 2024-12-10T11:04:08,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648857 2024-12-10T11:04:08,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648857 2024-12-10T11:04:08,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648857) roll requested 2024-12-10T11:04:08,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648864 2024-12-10T11:04:08,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648864 2024-12-10T11:04:08,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648864) roll requested 2024-12-10T11:04:08,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648872 2024-12-10T11:04:08,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648872 2024-12-10T11:04:08,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648872) roll requested 2024-12-10T11:04:08,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648879 2024-12-10T11:04:08,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648879 2024-12-10T11:04:08,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648879) roll requested 2024-12-10T11:04:08,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648890 2024-12-10T11:04:08,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648890 2024-12-10T11:04:08,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648890) roll requested 2024-12-10T11:04:08,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648908 2024-12-10T11:04:08,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648908 2024-12-10T11:04:08,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648908) roll requested 2024-12-10T11:04:08,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648918 2024-12-10T11:04:08,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648918 2024-12-10T11:04:08,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648918) roll requested 2024-12-10T11:04:08,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648934 2024-12-10T11:04:08,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648934 2024-12-10T11:04:08,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648934) roll requested 2024-12-10T11:04:08,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648943 2024-12-10T11:04:08,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648943 2024-12-10T11:04:08,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648943) roll requested 2024-12-10T11:04:08,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648955 2024-12-10T11:04:08,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648955 2024-12-10T11:04:08,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648955) roll requested 2024-12-10T11:04:08,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648974 2024-12-10T11:04:08,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648974 2024-12-10T11:04:08,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648974) roll requested 2024-12-10T11:04:08,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648986 2024-12-10T11:04:08,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:08,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648986 2024-12-10T11:04:08,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:08,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:08,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:08,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:08,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648986) roll requested 2024-12-10T11:04:08,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828648998 2024-12-10T11:04:09,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648998 2024-12-10T11:04:09,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828648998) roll requested 2024-12-10T11:04:09,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649022 2024-12-10T11:04:09,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828648998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649022 2024-12-10T11:04:09,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649022) roll requested 2024-12-10T11:04:09,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649033 2024-12-10T11:04:09,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649033 2024-12-10T11:04:09,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649033) roll requested 2024-12-10T11:04:09,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649041 2024-12-10T11:04:09,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649041 2024-12-10T11:04:09,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649041) roll requested 2024-12-10T11:04:09,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649049 2024-12-10T11:04:09,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649049 2024-12-10T11:04:09,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649049) roll requested 2024-12-10T11:04:09,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649056 2024-12-10T11:04:09,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649056 2024-12-10T11:04:09,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649056) roll requested 2024-12-10T11:04:09,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649063 2024-12-10T11:04:09,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649063 2024-12-10T11:04:09,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649063) roll requested 2024-12-10T11:04:09,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649070 2024-12-10T11:04:09,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649070 2024-12-10T11:04:09,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649070) roll requested 2024-12-10T11:04:09,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649078 2024-12-10T11:04:09,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649078 2024-12-10T11:04:09,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649078) roll requested 2024-12-10T11:04:09,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649084 2024-12-10T11:04:09,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649084 2024-12-10T11:04:09,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649084) roll requested 2024-12-10T11:04:09,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649091 2024-12-10T11:04:09,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649091 2024-12-10T11:04:09,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649091) roll requested 2024-12-10T11:04:09,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649097 2024-12-10T11:04:09,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649097 2024-12-10T11:04:09,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649097) roll requested 2024-12-10T11:04:09,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649104 2024-12-10T11:04:09,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649104 2024-12-10T11:04:09,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649104) roll requested 2024-12-10T11:04:09,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649110 2024-12-10T11:04:09,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649110 2024-12-10T11:04:09,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649110) roll requested 2024-12-10T11:04:09,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649116 2024-12-10T11:04:09,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649116 2024-12-10T11:04:09,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649116) roll requested 2024-12-10T11:04:09,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649123 2024-12-10T11:04:09,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649123 2024-12-10T11:04:09,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649123) roll requested 2024-12-10T11:04:09,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649130 2024-12-10T11:04:09,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649130 2024-12-10T11:04:09,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649130) roll requested 2024-12-10T11:04:09,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649136 2024-12-10T11:04:09,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649136 2024-12-10T11:04:09,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649136) roll requested 2024-12-10T11:04:09,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649142 2024-12-10T11:04:09,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649142 2024-12-10T11:04:09,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649142) roll requested 2024-12-10T11:04:09,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649149 2024-12-10T11:04:09,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649149 2024-12-10T11:04:09,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649149) roll requested 2024-12-10T11:04:09,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649155 2024-12-10T11:04:09,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649155 2024-12-10T11:04:09,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649155) roll requested 2024-12-10T11:04:09,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649162 2024-12-10T11:04:09,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649162 2024-12-10T11:04:09,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649162) roll requested 2024-12-10T11:04:09,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649170 2024-12-10T11:04:09,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649170 2024-12-10T11:04:09,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649170) roll requested 2024-12-10T11:04:09,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649177 2024-12-10T11:04:09,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649177 2024-12-10T11:04:09,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649177) roll requested 2024-12-10T11:04:09,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649183 2024-12-10T11:04:09,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649183 2024-12-10T11:04:09,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649183) roll requested 2024-12-10T11:04:09,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649190 2024-12-10T11:04:09,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649190 2024-12-10T11:04:09,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649190) roll requested 2024-12-10T11:04:09,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649196 2024-12-10T11:04:09,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649196 2024-12-10T11:04:09,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649196) roll requested 2024-12-10T11:04:09,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649203 2024-12-10T11:04:09,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649203 2024-12-10T11:04:09,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649203) roll requested 2024-12-10T11:04:09,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649210 2024-12-10T11:04:09,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649210 2024-12-10T11:04:09,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649210) roll requested 2024-12-10T11:04:09,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649216 2024-12-10T11:04:09,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649216 2024-12-10T11:04:09,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649216) roll requested 2024-12-10T11:04:09,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649222 2024-12-10T11:04:09,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649222 2024-12-10T11:04:09,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649222) roll requested 2024-12-10T11:04:09,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649229 2024-12-10T11:04:09,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649229 2024-12-10T11:04:09,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649229) roll requested 2024-12-10T11:04:09,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649236 2024-12-10T11:04:09,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649236 2024-12-10T11:04:09,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,242 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,242 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649236) roll requested 2024-12-10T11:04:09,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649242 2024-12-10T11:04:09,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649242 2024-12-10T11:04:09,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649242) roll requested 2024-12-10T11:04:09,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649248 2024-12-10T11:04:09,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649248 2024-12-10T11:04:09,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649248) roll requested 2024-12-10T11:04:09,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649255 2024-12-10T11:04:09,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649255 2024-12-10T11:04:09,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649255) roll requested 2024-12-10T11:04:09,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649262 2024-12-10T11:04:09,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649262 2024-12-10T11:04:09,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649262) roll requested 2024-12-10T11:04:09,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649268 2024-12-10T11:04:09,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649268 2024-12-10T11:04:09,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649268) roll requested 2024-12-10T11:04:09,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649275 2024-12-10T11:04:09,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649275 2024-12-10T11:04:09,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649275) roll requested 2024-12-10T11:04:09,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649281 2024-12-10T11:04:09,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649281 2024-12-10T11:04:09,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649281) roll requested 2024-12-10T11:04:09,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649288 2024-12-10T11:04:09,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649288 2024-12-10T11:04:09,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649288) roll requested 2024-12-10T11:04:09,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649294 2024-12-10T11:04:09,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649294 2024-12-10T11:04:09,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649294) roll requested 2024-12-10T11:04:09,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649300 2024-12-10T11:04:09,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649300 2024-12-10T11:04:09,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649300) roll requested 2024-12-10T11:04:09,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649307 2024-12-10T11:04:09,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649307 2024-12-10T11:04:09,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649307) roll requested 2024-12-10T11:04:09,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649313 2024-12-10T11:04:09,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649313 2024-12-10T11:04:09,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649313) roll requested 2024-12-10T11:04:09,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649320 2024-12-10T11:04:09,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649320 2024-12-10T11:04:09,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649320) roll requested 2024-12-10T11:04:09,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649326 2024-12-10T11:04:09,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649326 2024-12-10T11:04:09,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649326) roll requested 2024-12-10T11:04:09,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649335 2024-12-10T11:04:09,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649335 2024-12-10T11:04:09,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649335) roll requested 2024-12-10T11:04:09,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649342 2024-12-10T11:04:09,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649342 2024-12-10T11:04:09,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649342) roll requested 2024-12-10T11:04:09,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649349 2024-12-10T11:04:09,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649349 2024-12-10T11:04:09,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649349) roll requested 2024-12-10T11:04:09,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649355 2024-12-10T11:04:09,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649355 2024-12-10T11:04:09,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649355) roll requested 2024-12-10T11:04:09,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649362 2024-12-10T11:04:09,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649362 2024-12-10T11:04:09,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649362) roll requested 2024-12-10T11:04:09,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649368 2024-12-10T11:04:09,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649368 2024-12-10T11:04:09,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649368) roll requested 2024-12-10T11:04:09,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649375 2024-12-10T11:04:09,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649375 2024-12-10T11:04:09,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649375) roll requested 2024-12-10T11:04:09,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649381 2024-12-10T11:04:09,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649381 2024-12-10T11:04:09,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649381) roll requested 2024-12-10T11:04:09,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649388 2024-12-10T11:04:09,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649388 2024-12-10T11:04:09,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649388) roll requested 2024-12-10T11:04:09,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649395 2024-12-10T11:04:09,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649395 2024-12-10T11:04:09,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649395) roll requested 2024-12-10T11:04:09,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649401 2024-12-10T11:04:09,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649401 2024-12-10T11:04:09,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649401) roll requested 2024-12-10T11:04:09,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649407 2024-12-10T11:04:09,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649407 2024-12-10T11:04:09,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649407) roll requested 2024-12-10T11:04:09,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649414 2024-12-10T11:04:09,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649414 2024-12-10T11:04:09,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649414) roll requested 2024-12-10T11:04:09,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649420 2024-12-10T11:04:09,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649420 2024-12-10T11:04:09,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649420) roll requested 2024-12-10T11:04:09,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649427 2024-12-10T11:04:09,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649427 2024-12-10T11:04:09,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649427) roll requested 2024-12-10T11:04:09,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649433 2024-12-10T11:04:09,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649433 2024-12-10T11:04:09,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649433) roll requested 2024-12-10T11:04:09,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649440 2024-12-10T11:04:09,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649440 2024-12-10T11:04:09,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649440) roll requested 2024-12-10T11:04:09,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649446 2024-12-10T11:04:09,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649446 2024-12-10T11:04:09,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649446) roll requested 2024-12-10T11:04:09,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649453 2024-12-10T11:04:09,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649453 2024-12-10T11:04:09,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649453) roll requested 2024-12-10T11:04:09,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649459 2024-12-10T11:04:09,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649459 2024-12-10T11:04:09,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649459) roll requested 2024-12-10T11:04:09,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649465 2024-12-10T11:04:09,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649465 2024-12-10T11:04:09,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649465) roll requested 2024-12-10T11:04:09,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649472 2024-12-10T11:04:09,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649472 2024-12-10T11:04:09,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649472) roll requested 2024-12-10T11:04:09,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649478 2024-12-10T11:04:09,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649478 2024-12-10T11:04:09,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649478) roll requested 2024-12-10T11:04:09,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649485 2024-12-10T11:04:09,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649485 2024-12-10T11:04:09,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649485) roll requested 2024-12-10T11:04:09,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649492 2024-12-10T11:04:09,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649492 2024-12-10T11:04:09,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649492) roll requested 2024-12-10T11:04:09,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649500 2024-12-10T11:04:09,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649500 2024-12-10T11:04:09,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649500) roll requested 2024-12-10T11:04:09,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649508 2024-12-10T11:04:09,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649508 2024-12-10T11:04:09,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649508) roll requested 2024-12-10T11:04:09,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649515 2024-12-10T11:04:09,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649515 2024-12-10T11:04:09,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649515) roll requested 2024-12-10T11:04:09,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649526 2024-12-10T11:04:09,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649526 2024-12-10T11:04:09,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649526) roll requested 2024-12-10T11:04:09,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649535 2024-12-10T11:04:09,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649535 2024-12-10T11:04:09,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649535) roll requested 2024-12-10T11:04:09,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649543 2024-12-10T11:04:09,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649543 2024-12-10T11:04:09,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649543) roll requested 2024-12-10T11:04:09,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649552 2024-12-10T11:04:09,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649552 2024-12-10T11:04:09,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649552) roll requested 2024-12-10T11:04:09,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649560 2024-12-10T11:04:09,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649560 2024-12-10T11:04:09,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649560) roll requested 2024-12-10T11:04:09,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649569 2024-12-10T11:04:09,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649569 2024-12-10T11:04:09,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649569) roll requested 2024-12-10T11:04:09,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649578 2024-12-10T11:04:09,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649578 2024-12-10T11:04:09,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649578) roll requested 2024-12-10T11:04:09,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649585 2024-12-10T11:04:09,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649585 2024-12-10T11:04:09,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649585) roll requested 2024-12-10T11:04:09,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649593 2024-12-10T11:04:09,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649593 2024-12-10T11:04:09,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649593) roll requested 2024-12-10T11:04:09,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649604 2024-12-10T11:04:09,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649604 2024-12-10T11:04:09,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649604) roll requested 2024-12-10T11:04:09,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649612 2024-12-10T11:04:09,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649612 2024-12-10T11:04:09,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649612) roll requested 2024-12-10T11:04:09,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649622 2024-12-10T11:04:09,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649622 2024-12-10T11:04:09,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649622) roll requested 2024-12-10T11:04:09,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649629 2024-12-10T11:04:09,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649629 2024-12-10T11:04:09,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649629) roll requested 2024-12-10T11:04:09,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649636 2024-12-10T11:04:09,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649636 2024-12-10T11:04:09,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649636) roll requested 2024-12-10T11:04:09,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649646 2024-12-10T11:04:09,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649646 2024-12-10T11:04:09,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649646) roll requested 2024-12-10T11:04:09,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649658 2024-12-10T11:04:09,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649658 2024-12-10T11:04:09,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649658) roll requested 2024-12-10T11:04:09,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649668 2024-12-10T11:04:09,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649668 2024-12-10T11:04:09,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649668) roll requested 2024-12-10T11:04:09,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649679 2024-12-10T11:04:09,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649679 2024-12-10T11:04:09,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649679) roll requested 2024-12-10T11:04:09,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649688 2024-12-10T11:04:09,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649688 2024-12-10T11:04:09,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649688) roll requested 2024-12-10T11:04:09,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649696 2024-12-10T11:04:09,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649696 2024-12-10T11:04:09,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649696) roll requested 2024-12-10T11:04:09,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649704 2024-12-10T11:04:09,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649704 2024-12-10T11:04:09,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649704) roll requested 2024-12-10T11:04:09,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649713 2024-12-10T11:04:09,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649713 2024-12-10T11:04:09,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,723 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649713) roll requested 2024-12-10T11:04:09,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649723 2024-12-10T11:04:09,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649723 2024-12-10T11:04:09,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649723) roll requested 2024-12-10T11:04:09,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649731 2024-12-10T11:04:09,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649731 2024-12-10T11:04:09,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649731) roll requested 2024-12-10T11:04:09,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649741 2024-12-10T11:04:09,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649741 2024-12-10T11:04:09,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649741) roll requested 2024-12-10T11:04:09,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649749 2024-12-10T11:04:09,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649749 2024-12-10T11:04:09,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649749) roll requested 2024-12-10T11:04:09,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649758 2024-12-10T11:04:09,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649758 2024-12-10T11:04:09,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649758) roll requested 2024-12-10T11:04:09,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649766 2024-12-10T11:04:09,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649766 2024-12-10T11:04:09,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649766) roll requested 2024-12-10T11:04:09,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649772 2024-12-10T11:04:09,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649772 2024-12-10T11:04:09,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649772) roll requested 2024-12-10T11:04:09,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649779 2024-12-10T11:04:09,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649779 2024-12-10T11:04:09,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649779) roll requested 2024-12-10T11:04:09,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649785 2024-12-10T11:04:09,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649785 2024-12-10T11:04:09,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649785) roll requested 2024-12-10T11:04:09,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649792 2024-12-10T11:04:09,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649792 2024-12-10T11:04:09,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,798 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649792) roll requested 2024-12-10T11:04:09,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649798 2024-12-10T11:04:09,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649798 2024-12-10T11:04:09,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649798) roll requested 2024-12-10T11:04:09,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649804 2024-12-10T11:04:09,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649804 2024-12-10T11:04:09,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649804) roll requested 2024-12-10T11:04:09,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649810 2024-12-10T11:04:09,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649810 2024-12-10T11:04:09,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649810) roll requested 2024-12-10T11:04:09,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649817 2024-12-10T11:04:09,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649817 2024-12-10T11:04:09,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649817) roll requested 2024-12-10T11:04:09,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649823 2024-12-10T11:04:09,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649823 2024-12-10T11:04:09,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649823) roll requested 2024-12-10T11:04:09,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649830 2024-12-10T11:04:09,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649830 2024-12-10T11:04:09,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649830) roll requested 2024-12-10T11:04:09,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649836 2024-12-10T11:04:09,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649836 2024-12-10T11:04:09,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649836) roll requested 2024-12-10T11:04:09,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649843 2024-12-10T11:04:09,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649843 2024-12-10T11:04:09,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649843) roll requested 2024-12-10T11:04:09,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649849 2024-12-10T11:04:09,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649849 2024-12-10T11:04:09,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649849) roll requested 2024-12-10T11:04:09,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649856 2024-12-10T11:04:09,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649856 2024-12-10T11:04:09,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649856) roll requested 2024-12-10T11:04:09,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649863 2024-12-10T11:04:09,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649863 2024-12-10T11:04:09,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649863) roll requested 2024-12-10T11:04:09,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649870 2024-12-10T11:04:09,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649870 2024-12-10T11:04:09,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649870) roll requested 2024-12-10T11:04:09,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649877 2024-12-10T11:04:09,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649877 2024-12-10T11:04:09,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649877) roll requested 2024-12-10T11:04:09,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649885 2024-12-10T11:04:09,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649885 2024-12-10T11:04:09,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649885) roll requested 2024-12-10T11:04:09,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649893 2024-12-10T11:04:09,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649893 2024-12-10T11:04:09,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649893) roll requested 2024-12-10T11:04:09,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649914 2024-12-10T11:04:09,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649914 2024-12-10T11:04:09,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649914) roll requested 2024-12-10T11:04:09,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649930 2024-12-10T11:04:09,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649930 2024-12-10T11:04:09,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649930) roll requested 2024-12-10T11:04:09,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649946 2024-12-10T11:04:09,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649946 2024-12-10T11:04:09,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649946) roll requested 2024-12-10T11:04:09,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649955 2024-12-10T11:04:09,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649955 2024-12-10T11:04:09,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649955) roll requested 2024-12-10T11:04:09,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649964 2024-12-10T11:04:09,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649964 2024-12-10T11:04:09,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649964) roll requested 2024-12-10T11:04:09,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649971 2024-12-10T11:04:09,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649971 2024-12-10T11:04:09,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649971) roll requested 2024-12-10T11:04:09,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649981 2024-12-10T11:04:09,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:09,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649981 2024-12-10T11:04:09,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:09,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:09,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:09,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:09,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649981) roll requested 2024-12-10T11:04:09,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828649997 2024-12-10T11:04:10,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649997 2024-12-10T11:04:10,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828649997) roll requested 2024-12-10T11:04:10,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650005 2024-12-10T11:04:10,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828649997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650005 2024-12-10T11:04:10,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650005) roll requested 2024-12-10T11:04:10,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650012 2024-12-10T11:04:10,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650012 2024-12-10T11:04:10,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650012) roll requested 2024-12-10T11:04:10,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650021 2024-12-10T11:04:10,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650021 2024-12-10T11:04:10,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650021) roll requested 2024-12-10T11:04:10,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650029 2024-12-10T11:04:10,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650029 2024-12-10T11:04:10,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650029) roll requested 2024-12-10T11:04:10,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650037 2024-12-10T11:04:10,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650037 2024-12-10T11:04:10,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650037) roll requested 2024-12-10T11:04:10,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650045 2024-12-10T11:04:10,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650045 2024-12-10T11:04:10,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650045) roll requested 2024-12-10T11:04:10,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650054 2024-12-10T11:04:10,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650054 2024-12-10T11:04:10,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650054) roll requested 2024-12-10T11:04:10,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650065 2024-12-10T11:04:10,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650065 2024-12-10T11:04:10,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650065) roll requested 2024-12-10T11:04:10,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650073 2024-12-10T11:04:10,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650073 2024-12-10T11:04:10,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650073) roll requested 2024-12-10T11:04:10,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650080 2024-12-10T11:04:10,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650080 2024-12-10T11:04:10,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650080) roll requested 2024-12-10T11:04:10,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650087 2024-12-10T11:04:10,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650087 2024-12-10T11:04:10,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650087) roll requested 2024-12-10T11:04:10,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650093 2024-12-10T11:04:10,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650093 2024-12-10T11:04:10,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650093) roll requested 2024-12-10T11:04:10,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650100 2024-12-10T11:04:10,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650100 2024-12-10T11:04:10,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650100) roll requested 2024-12-10T11:04:10,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650106 2024-12-10T11:04:10,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650106 2024-12-10T11:04:10,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650106) roll requested 2024-12-10T11:04:10,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650113 2024-12-10T11:04:10,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650113 2024-12-10T11:04:10,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650113) roll requested 2024-12-10T11:04:10,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650120 2024-12-10T11:04:10,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650120 2024-12-10T11:04:10,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650120) roll requested 2024-12-10T11:04:10,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650127 2024-12-10T11:04:10,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650127 2024-12-10T11:04:10,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650127) roll requested 2024-12-10T11:04:10,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650133 2024-12-10T11:04:10,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650133 2024-12-10T11:04:10,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650133) roll requested 2024-12-10T11:04:10,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650140 2024-12-10T11:04:10,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650140 2024-12-10T11:04:10,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650140) roll requested 2024-12-10T11:04:10,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650146 2024-12-10T11:04:10,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650146 2024-12-10T11:04:10,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650146) roll requested 2024-12-10T11:04:10,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650153 2024-12-10T11:04:10,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650153 2024-12-10T11:04:10,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650153) roll requested 2024-12-10T11:04:10,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650160 2024-12-10T11:04:10,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650160 2024-12-10T11:04:10,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650160) roll requested 2024-12-10T11:04:10,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650166 2024-12-10T11:04:10,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650166 2024-12-10T11:04:10,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650166) roll requested 2024-12-10T11:04:10,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650173 2024-12-10T11:04:10,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650173 2024-12-10T11:04:10,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650173) roll requested 2024-12-10T11:04:10,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650180 2024-12-10T11:04:10,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650180 2024-12-10T11:04:10,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650180) roll requested 2024-12-10T11:04:10,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650189 2024-12-10T11:04:10,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650189 2024-12-10T11:04:10,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650189) roll requested 2024-12-10T11:04:10,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650196 2024-12-10T11:04:10,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650196 2024-12-10T11:04:10,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650196) roll requested 2024-12-10T11:04:10,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650210 2024-12-10T11:04:10,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650210 2024-12-10T11:04:10,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650210) roll requested 2024-12-10T11:04:10,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650218 2024-12-10T11:04:10,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650218 2024-12-10T11:04:10,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650218) roll requested 2024-12-10T11:04:10,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650226 2024-12-10T11:04:10,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650226 2024-12-10T11:04:10,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650226) roll requested 2024-12-10T11:04:10,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650233 2024-12-10T11:04:10,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650233 2024-12-10T11:04:10,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650233) roll requested 2024-12-10T11:04:10,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650241 2024-12-10T11:04:10,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650241 2024-12-10T11:04:10,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650241) roll requested 2024-12-10T11:04:10,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650249 2024-12-10T11:04:10,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650249 2024-12-10T11:04:10,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650249) roll requested 2024-12-10T11:04:10,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650257 2024-12-10T11:04:10,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650257 2024-12-10T11:04:10,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650257) roll requested 2024-12-10T11:04:10,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650268 2024-12-10T11:04:10,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650268 2024-12-10T11:04:10,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650268) roll requested 2024-12-10T11:04:10,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650277 2024-12-10T11:04:10,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650277 2024-12-10T11:04:10,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650277) roll requested 2024-12-10T11:04:10,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650285 2024-12-10T11:04:10,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650285 2024-12-10T11:04:10,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650285) roll requested 2024-12-10T11:04:10,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650293 2024-12-10T11:04:10,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650293 2024-12-10T11:04:10,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650293) roll requested 2024-12-10T11:04:10,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650304 2024-12-10T11:04:10,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650304 2024-12-10T11:04:10,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650304) roll requested 2024-12-10T11:04:10,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650314 2024-12-10T11:04:10,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650314 2024-12-10T11:04:10,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650314) roll requested 2024-12-10T11:04:10,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650327 2024-12-10T11:04:10,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650327 2024-12-10T11:04:10,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650327) roll requested 2024-12-10T11:04:10,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650342 2024-12-10T11:04:10,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650342 2024-12-10T11:04:10,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650342) roll requested 2024-12-10T11:04:10,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650350 2024-12-10T11:04:10,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650350 2024-12-10T11:04:10,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650350) roll requested 2024-12-10T11:04:10,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650357 2024-12-10T11:04:10,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650357 2024-12-10T11:04:10,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650357) roll requested 2024-12-10T11:04:10,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650370 2024-12-10T11:04:10,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650370 2024-12-10T11:04:10,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650370) roll requested 2024-12-10T11:04:10,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650378 2024-12-10T11:04:10,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650378 2024-12-10T11:04:10,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650378) roll requested 2024-12-10T11:04:10,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650385 2024-12-10T11:04:10,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650385 2024-12-10T11:04:10,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650385) roll requested 2024-12-10T11:04:10,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650395 2024-12-10T11:04:10,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650395 2024-12-10T11:04:10,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650395) roll requested 2024-12-10T11:04:10,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650402 2024-12-10T11:04:10,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650402 2024-12-10T11:04:10,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650402) roll requested 2024-12-10T11:04:10,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650413 2024-12-10T11:04:10,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650413 2024-12-10T11:04:10,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650413) roll requested 2024-12-10T11:04:10,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650421 2024-12-10T11:04:10,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650421 2024-12-10T11:04:10,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650421) roll requested 2024-12-10T11:04:10,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650428 2024-12-10T11:04:10,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650428 2024-12-10T11:04:10,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650428) roll requested 2024-12-10T11:04:10,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650435 2024-12-10T11:04:10,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650435 2024-12-10T11:04:10,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650435) roll requested 2024-12-10T11:04:10,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650442 2024-12-10T11:04:10,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650442 2024-12-10T11:04:10,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650442) roll requested 2024-12-10T11:04:10,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650450 2024-12-10T11:04:10,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650450 2024-12-10T11:04:10,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650450) roll requested 2024-12-10T11:04:10,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650457 2024-12-10T11:04:10,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650457 2024-12-10T11:04:10,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650457) roll requested 2024-12-10T11:04:10,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650464 2024-12-10T11:04:10,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650464 2024-12-10T11:04:10,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650464) roll requested 2024-12-10T11:04:10,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650471 2024-12-10T11:04:10,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650471 2024-12-10T11:04:10,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650471) roll requested 2024-12-10T11:04:10,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650481 2024-12-10T11:04:10,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650481 2024-12-10T11:04:10,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650481) roll requested 2024-12-10T11:04:10,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650490 2024-12-10T11:04:10,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650490 2024-12-10T11:04:10,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650481 is not closed yet, will try archiving it next time 2024-12-10T11:04:10,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650490) roll requested 2024-12-10T11:04:10,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650497 2024-12-10T11:04:10,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650497 2024-12-10T11:04:10,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650497) roll requested 2024-12-10T11:04:10,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650504 2024-12-10T11:04:10,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650504 2024-12-10T11:04:10,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650504) roll requested 2024-12-10T11:04:10,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650511 2024-12-10T11:04:10,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650511 2024-12-10T11:04:10,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650511) roll requested 2024-12-10T11:04:10,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650518 2024-12-10T11:04:10,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650518 2024-12-10T11:04:10,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650518) roll requested 2024-12-10T11:04:10,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650528 2024-12-10T11:04:10,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650528 2024-12-10T11:04:10,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650528) roll requested 2024-12-10T11:04:10,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650535 2024-12-10T11:04:10,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650535 2024-12-10T11:04:10,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650535) roll requested 2024-12-10T11:04:10,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650543 2024-12-10T11:04:10,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650543 2024-12-10T11:04:10,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650543) roll requested 2024-12-10T11:04:10,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650554 2024-12-10T11:04:10,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650554 2024-12-10T11:04:10,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650554) roll requested 2024-12-10T11:04:10,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650564 2024-12-10T11:04:10,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650564 2024-12-10T11:04:10,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650564) roll requested 2024-12-10T11:04:10,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650572 2024-12-10T11:04:10,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650572 2024-12-10T11:04:10,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650572) roll requested 2024-12-10T11:04:10,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650578 2024-12-10T11:04:10,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650578 2024-12-10T11:04:10,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650578) roll requested 2024-12-10T11:04:10,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650585 2024-12-10T11:04:10,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650585 2024-12-10T11:04:10,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650585) roll requested 2024-12-10T11:04:10,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650592 2024-12-10T11:04:10,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650592 2024-12-10T11:04:10,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650592) roll requested 2024-12-10T11:04:10,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650598 2024-12-10T11:04:10,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650598 2024-12-10T11:04:10,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650598) roll requested 2024-12-10T11:04:10,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650604 2024-12-10T11:04:10,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650604 2024-12-10T11:04:10,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650604) roll requested 2024-12-10T11:04:10,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650611 2024-12-10T11:04:10,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650611 2024-12-10T11:04:10,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650611) roll requested 2024-12-10T11:04:10,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650618 2024-12-10T11:04:10,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650618 2024-12-10T11:04:10,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650618) roll requested 2024-12-10T11:04:10,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650625 2024-12-10T11:04:10,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650625 2024-12-10T11:04:10,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650625) roll requested 2024-12-10T11:04:10,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650632 2024-12-10T11:04:10,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650632 2024-12-10T11:04:10,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650632) roll requested 2024-12-10T11:04:10,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650640 2024-12-10T11:04:10,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650640 2024-12-10T11:04:10,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650640) roll requested 2024-12-10T11:04:10,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650647 2024-12-10T11:04:10,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650647 2024-12-10T11:04:10,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650647) roll requested 2024-12-10T11:04:10,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650653 2024-12-10T11:04:10,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650653 2024-12-10T11:04:10,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650653) roll requested 2024-12-10T11:04:10,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650661 2024-12-10T11:04:10,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650661 2024-12-10T11:04:10,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650661) roll requested 2024-12-10T11:04:10,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650669 2024-12-10T11:04:10,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650669 2024-12-10T11:04:10,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650669) roll requested 2024-12-10T11:04:10,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650675 2024-12-10T11:04:10,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650675 2024-12-10T11:04:10,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650675) roll requested 2024-12-10T11:04:10,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650682 2024-12-10T11:04:10,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650682 2024-12-10T11:04:10,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650682) roll requested 2024-12-10T11:04:10,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650690 2024-12-10T11:04:10,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650690 2024-12-10T11:04:10,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650690) roll requested 2024-12-10T11:04:10,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650697 2024-12-10T11:04:10,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650697 2024-12-10T11:04:10,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650697) roll requested 2024-12-10T11:04:10,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650705 2024-12-10T11:04:10,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650705 2024-12-10T11:04:10,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650705) roll requested 2024-12-10T11:04:10,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650712 2024-12-10T11:04:10,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650712 2024-12-10T11:04:10,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650712) roll requested 2024-12-10T11:04:10,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650730 2024-12-10T11:04:10,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650730 2024-12-10T11:04:10,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650730) roll requested 2024-12-10T11:04:10,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650747 2024-12-10T11:04:10,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650747 2024-12-10T11:04:10,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650747) roll requested 2024-12-10T11:04:10,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650778 2024-12-10T11:04:10,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650778 2024-12-10T11:04:10,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650778) roll requested 2024-12-10T11:04:10,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650806 2024-12-10T11:04:10,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650806 2024-12-10T11:04:10,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650806) roll requested 2024-12-10T11:04:10,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650822 2024-12-10T11:04:10,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650822 2024-12-10T11:04:10,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650822) roll requested 2024-12-10T11:04:10,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650830 2024-12-10T11:04:10,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650830 2024-12-10T11:04:10,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650830) roll requested 2024-12-10T11:04:10,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650837 2024-12-10T11:04:10,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650837 2024-12-10T11:04:10,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650837) roll requested 2024-12-10T11:04:10,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650846 2024-12-10T11:04:10,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650846 2024-12-10T11:04:10,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650846) roll requested 2024-12-10T11:04:10,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650858 2024-12-10T11:04:10,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650858 2024-12-10T11:04:10,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650858) roll requested 2024-12-10T11:04:10,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650866 2024-12-10T11:04:10,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650866 2024-12-10T11:04:10,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650866) roll requested 2024-12-10T11:04:10,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650874 2024-12-10T11:04:10,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650874 2024-12-10T11:04:10,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650874) roll requested 2024-12-10T11:04:10,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650883 2024-12-10T11:04:10,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650883 2024-12-10T11:04:10,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650883) roll requested 2024-12-10T11:04:10,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650890 2024-12-10T11:04:10,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650890 2024-12-10T11:04:10,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650890) roll requested 2024-12-10T11:04:10,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650898 2024-12-10T11:04:10,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650898 2024-12-10T11:04:10,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650898) roll requested 2024-12-10T11:04:10,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650909 2024-12-10T11:04:10,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650909 2024-12-10T11:04:10,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650909) roll requested 2024-12-10T11:04:10,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650918 2024-12-10T11:04:10,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650918 2024-12-10T11:04:10,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650918) roll requested 2024-12-10T11:04:10,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650930 2024-12-10T11:04:10,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650930 2024-12-10T11:04:10,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650930) roll requested 2024-12-10T11:04:10,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650942 2024-12-10T11:04:10,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650942 2024-12-10T11:04:10,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650942) roll requested 2024-12-10T11:04:10,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650949 2024-12-10T11:04:10,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650949 2024-12-10T11:04:10,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650949) roll requested 2024-12-10T11:04:10,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650957 2024-12-10T11:04:10,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650957 2024-12-10T11:04:10,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650957) roll requested 2024-12-10T11:04:10,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650964 2024-12-10T11:04:10,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650964 2024-12-10T11:04:10,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650964) roll requested 2024-12-10T11:04:10,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650973 2024-12-10T11:04:10,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650973 2024-12-10T11:04:10,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650973) roll requested 2024-12-10T11:04:10,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650981 2024-12-10T11:04:10,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650981 2024-12-10T11:04:10,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:10,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:10,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650981) roll requested 2024-12-10T11:04:10,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828650989 2024-12-10T11:04:10,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:10,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650989 2024-12-10T11:04:10,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:10,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:10,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828650989) roll requested 2024-12-10T11:04:11,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651000 2024-12-10T11:04:11,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828650989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651000 2024-12-10T11:04:11,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651000) roll requested 2024-12-10T11:04:11,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651007 2024-12-10T11:04:11,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651007 2024-12-10T11:04:11,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651007) roll requested 2024-12-10T11:04:11,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651014 2024-12-10T11:04:11,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651014 2024-12-10T11:04:11,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651014) roll requested 2024-12-10T11:04:11,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651022 2024-12-10T11:04:11,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651022 2024-12-10T11:04:11,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651022) roll requested 2024-12-10T11:04:11,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651030 2024-12-10T11:04:11,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651030 2024-12-10T11:04:11,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651030) roll requested 2024-12-10T11:04:11,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651037 2024-12-10T11:04:11,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651037 2024-12-10T11:04:11,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651037) roll requested 2024-12-10T11:04:11,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651044 2024-12-10T11:04:11,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651044 2024-12-10T11:04:11,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651044) roll requested 2024-12-10T11:04:11,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651057 2024-12-10T11:04:11,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651057 2024-12-10T11:04:11,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651057) roll requested 2024-12-10T11:04:11,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651070 2024-12-10T11:04:11,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651070 2024-12-10T11:04:11,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651070) roll requested 2024-12-10T11:04:11,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651082 2024-12-10T11:04:11,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651082 2024-12-10T11:04:11,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651082) roll requested 2024-12-10T11:04:11,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651092 2024-12-10T11:04:11,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651092 2024-12-10T11:04:11,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651092) roll requested 2024-12-10T11:04:11,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651100 2024-12-10T11:04:11,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651100 2024-12-10T11:04:11,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651100) roll requested 2024-12-10T11:04:11,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651108 2024-12-10T11:04:11,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651108 2024-12-10T11:04:11,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651108) roll requested 2024-12-10T11:04:11,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651117 2024-12-10T11:04:11,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651117 2024-12-10T11:04:11,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651117) roll requested 2024-12-10T11:04:11,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651124 2024-12-10T11:04:11,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651124 2024-12-10T11:04:11,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651124) roll requested 2024-12-10T11:04:11,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651133 2024-12-10T11:04:11,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651133 2024-12-10T11:04:11,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651133) roll requested 2024-12-10T11:04:11,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651142 2024-12-10T11:04:11,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651142 2024-12-10T11:04:11,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651142) roll requested 2024-12-10T11:04:11,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651154 2024-12-10T11:04:11,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651154 2024-12-10T11:04:11,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651154) roll requested 2024-12-10T11:04:11,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651162 2024-12-10T11:04:11,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651162 2024-12-10T11:04:11,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651162) roll requested 2024-12-10T11:04:11,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651175 2024-12-10T11:04:11,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651175 2024-12-10T11:04:11,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651175) roll requested 2024-12-10T11:04:11,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651190 2024-12-10T11:04:11,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651190 2024-12-10T11:04:11,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651190) roll requested 2024-12-10T11:04:11,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651202 2024-12-10T11:04:11,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651202 2024-12-10T11:04:11,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651202) roll requested 2024-12-10T11:04:11,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651210 2024-12-10T11:04:11,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651210 2024-12-10T11:04:11,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651210) roll requested 2024-12-10T11:04:11,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651224 2024-12-10T11:04:11,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651224 2024-12-10T11:04:11,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651224) roll requested 2024-12-10T11:04:11,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651258 2024-12-10T11:04:11,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651258 2024-12-10T11:04:11,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651258) roll requested 2024-12-10T11:04:11,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651274 2024-12-10T11:04:11,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651274 2024-12-10T11:04:11,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651274) roll requested 2024-12-10T11:04:11,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651286 2024-12-10T11:04:11,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651286 2024-12-10T11:04:11,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651286) roll requested 2024-12-10T11:04:11,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651295 2024-12-10T11:04:11,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651295 2024-12-10T11:04:11,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651295) roll requested 2024-12-10T11:04:11,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651303 2024-12-10T11:04:11,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651303 2024-12-10T11:04:11,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651303) roll requested 2024-12-10T11:04:11,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651314 2024-12-10T11:04:11,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651314 2024-12-10T11:04:11,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,330 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651314) roll requested 2024-12-10T11:04:11,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651330 2024-12-10T11:04:11,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651330 2024-12-10T11:04:11,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651330) roll requested 2024-12-10T11:04:11,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651342 2024-12-10T11:04:11,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651330 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651342 2024-12-10T11:04:11,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651342) roll requested 2024-12-10T11:04:11,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651350 2024-12-10T11:04:11,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651350 2024-12-10T11:04:11,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651350) roll requested 2024-12-10T11:04:11,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651359 2024-12-10T11:04:11,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651359 2024-12-10T11:04:11,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651359) roll requested 2024-12-10T11:04:11,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651368 2024-12-10T11:04:11,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651368 2024-12-10T11:04:11,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651368) roll requested 2024-12-10T11:04:11,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651377 2024-12-10T11:04:11,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651377 2024-12-10T11:04:11,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651377) roll requested 2024-12-10T11:04:11,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651386 2024-12-10T11:04:11,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651386 2024-12-10T11:04:11,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651386) roll requested 2024-12-10T11:04:11,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651392 2024-12-10T11:04:11,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651392 2024-12-10T11:04:11,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651392) roll requested 2024-12-10T11:04:11,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651399 2024-12-10T11:04:11,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651399 2024-12-10T11:04:11,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651399) roll requested 2024-12-10T11:04:11,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651405 2024-12-10T11:04:11,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651405 2024-12-10T11:04:11,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651405) roll requested 2024-12-10T11:04:11,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651412 2024-12-10T11:04:11,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651412 2024-12-10T11:04:11,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651412) roll requested 2024-12-10T11:04:11,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651422 2024-12-10T11:04:11,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651422 2024-12-10T11:04:11,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651422) roll requested 2024-12-10T11:04:11,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651431 2024-12-10T11:04:11,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651422 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651431 2024-12-10T11:04:11,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651431) roll requested 2024-12-10T11:04:11,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651444 2024-12-10T11:04:11,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651444 2024-12-10T11:04:11,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651444) roll requested 2024-12-10T11:04:11,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651451 2024-12-10T11:04:11,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651451 2024-12-10T11:04:11,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651451) roll requested 2024-12-10T11:04:11,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651462 2024-12-10T11:04:11,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651462 2024-12-10T11:04:11,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651462) roll requested 2024-12-10T11:04:11,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651469 2024-12-10T11:04:11,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651469 2024-12-10T11:04:11,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651469) roll requested 2024-12-10T11:04:11,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651478 2024-12-10T11:04:11,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651478 2024-12-10T11:04:11,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651478) roll requested 2024-12-10T11:04:11,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651486 2024-12-10T11:04:11,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651486 2024-12-10T11:04:11,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,494 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651486) roll requested 2024-12-10T11:04:11,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651495 2024-12-10T11:04:11,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651495 2024-12-10T11:04:11,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651495) roll requested 2024-12-10T11:04:11,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651501 2024-12-10T11:04:11,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651501 2024-12-10T11:04:11,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651501) roll requested 2024-12-10T11:04:11,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651510 2024-12-10T11:04:11,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651510 2024-12-10T11:04:11,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651510) roll requested 2024-12-10T11:04:11,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651516 2024-12-10T11:04:11,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651516 2024-12-10T11:04:11,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651516) roll requested 2024-12-10T11:04:11,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651523 2024-12-10T11:04:11,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651523 2024-12-10T11:04:11,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651523) roll requested 2024-12-10T11:04:11,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651529 2024-12-10T11:04:11,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651529 2024-12-10T11:04:11,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651529) roll requested 2024-12-10T11:04:11,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651536 2024-12-10T11:04:11,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651536 2024-12-10T11:04:11,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651536) roll requested 2024-12-10T11:04:11,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651543 2024-12-10T11:04:11,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651543 2024-12-10T11:04:11,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651543) roll requested 2024-12-10T11:04:11,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651549 2024-12-10T11:04:11,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651549 2024-12-10T11:04:11,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651549) roll requested 2024-12-10T11:04:11,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651555 2024-12-10T11:04:11,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651555 2024-12-10T11:04:11,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651555) roll requested 2024-12-10T11:04:11,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651561 2024-12-10T11:04:11,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651561 2024-12-10T11:04:11,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651561) roll requested 2024-12-10T11:04:11,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651568 2024-12-10T11:04:11,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651568 2024-12-10T11:04:11,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,574 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651568) roll requested 2024-12-10T11:04:11,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651575 2024-12-10T11:04:11,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651575 2024-12-10T11:04:11,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651575) roll requested 2024-12-10T11:04:11,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651581 2024-12-10T11:04:11,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651581 2024-12-10T11:04:11,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651581) roll requested 2024-12-10T11:04:11,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651588 2024-12-10T11:04:11,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651588 2024-12-10T11:04:11,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651588) roll requested 2024-12-10T11:04:11,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651594 2024-12-10T11:04:11,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651594 2024-12-10T11:04:11,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651594) roll requested 2024-12-10T11:04:11,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651600 2024-12-10T11:04:11,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651600 2024-12-10T11:04:11,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651600) roll requested 2024-12-10T11:04:11,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651608 2024-12-10T11:04:11,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651608 2024-12-10T11:04:11,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651608) roll requested 2024-12-10T11:04:11,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651621 2024-12-10T11:04:11,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651621 2024-12-10T11:04:11,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651621) roll requested 2024-12-10T11:04:11,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651628 2024-12-10T11:04:11,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651628 2024-12-10T11:04:11,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651628) roll requested 2024-12-10T11:04:11,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651636 2024-12-10T11:04:11,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651636 2024-12-10T11:04:11,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651636) roll requested 2024-12-10T11:04:11,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651643 2024-12-10T11:04:11,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651643 2024-12-10T11:04:11,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651643) roll requested 2024-12-10T11:04:11,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651649 2024-12-10T11:04:11,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651649 2024-12-10T11:04:11,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651649) roll requested 2024-12-10T11:04:11,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651656 2024-12-10T11:04:11,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651656 2024-12-10T11:04:11,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651656) roll requested 2024-12-10T11:04:11,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651662 2024-12-10T11:04:11,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651662 2024-12-10T11:04:11,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651662) roll requested 2024-12-10T11:04:11,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651669 2024-12-10T11:04:11,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651669 2024-12-10T11:04:11,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651669) roll requested 2024-12-10T11:04:11,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651676 2024-12-10T11:04:11,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651676 2024-12-10T11:04:11,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651676) roll requested 2024-12-10T11:04:11,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651683 2024-12-10T11:04:11,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651683 2024-12-10T11:04:11,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651683) roll requested 2024-12-10T11:04:11,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651690 2024-12-10T11:04:11,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651690 2024-12-10T11:04:11,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651690) roll requested 2024-12-10T11:04:11,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651697 2024-12-10T11:04:11,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651697 2024-12-10T11:04:11,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651697) roll requested 2024-12-10T11:04:11,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651704 2024-12-10T11:04:11,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651704 2024-12-10T11:04:11,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651704) roll requested 2024-12-10T11:04:11,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651710 2024-12-10T11:04:11,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651710 2024-12-10T11:04:11,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651710) roll requested 2024-12-10T11:04:11,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651717 2024-12-10T11:04:11,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651717 2024-12-10T11:04:11,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651717) roll requested 2024-12-10T11:04:11,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651724 2024-12-10T11:04:11,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651724 2024-12-10T11:04:11,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651724) roll requested 2024-12-10T11:04:11,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651730 2024-12-10T11:04:11,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651730 2024-12-10T11:04:11,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651730) roll requested 2024-12-10T11:04:11,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651738 2024-12-10T11:04:11,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651738 2024-12-10T11:04:11,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651738) roll requested 2024-12-10T11:04:11,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651749 2024-12-10T11:04:11,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651749 2024-12-10T11:04:11,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651749) roll requested 2024-12-10T11:04:11,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651755 2024-12-10T11:04:11,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651755 2024-12-10T11:04:11,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,762 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,762 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651755) roll requested 2024-12-10T11:04:11,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651762 2024-12-10T11:04:11,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651762 2024-12-10T11:04:11,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651762) roll requested 2024-12-10T11:04:11,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651768 2024-12-10T11:04:11,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651762 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651768 2024-12-10T11:04:11,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651768) roll requested 2024-12-10T11:04:11,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651776 2024-12-10T11:04:11,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651776 2024-12-10T11:04:11,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651776) roll requested 2024-12-10T11:04:11,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651783 2024-12-10T11:04:11,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651783 2024-12-10T11:04:11,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,790 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651783) roll requested 2024-12-10T11:04:11,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651790 2024-12-10T11:04:11,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651790 2024-12-10T11:04:11,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651790) roll requested 2024-12-10T11:04:11,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651796 2024-12-10T11:04:11,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651796 2024-12-10T11:04:11,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651796) roll requested 2024-12-10T11:04:11,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651803 2024-12-10T11:04:11,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651803 2024-12-10T11:04:11,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651803) roll requested 2024-12-10T11:04:11,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651809 2024-12-10T11:04:11,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651809 2024-12-10T11:04:11,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651809) roll requested 2024-12-10T11:04:11,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651816 2024-12-10T11:04:11,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651816 2024-12-10T11:04:11,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651816) roll requested 2024-12-10T11:04:11,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651823 2024-12-10T11:04:11,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651823 2024-12-10T11:04:11,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651823) roll requested 2024-12-10T11:04:11,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651829 2024-12-10T11:04:11,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651829 2024-12-10T11:04:11,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651829) roll requested 2024-12-10T11:04:11,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651836 2024-12-10T11:04:11,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651836 2024-12-10T11:04:11,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651836) roll requested 2024-12-10T11:04:11,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651843 2024-12-10T11:04:11,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651843 2024-12-10T11:04:11,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651843) roll requested 2024-12-10T11:04:11,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651850 2024-12-10T11:04:11,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651850 2024-12-10T11:04:11,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651850) roll requested 2024-12-10T11:04:11,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651856 2024-12-10T11:04:11,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651856 2024-12-10T11:04:11,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651856) roll requested 2024-12-10T11:04:11,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651863 2024-12-10T11:04:11,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651863 2024-12-10T11:04:11,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651863) roll requested 2024-12-10T11:04:11,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651875 2024-12-10T11:04:11,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651875 2024-12-10T11:04:11,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651875) roll requested 2024-12-10T11:04:11,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651881 2024-12-10T11:04:11,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651881 2024-12-10T11:04:11,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651881) roll requested 2024-12-10T11:04:11,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651887 2024-12-10T11:04:11,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651887 2024-12-10T11:04:11,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651887) roll requested 2024-12-10T11:04:11,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651894 2024-12-10T11:04:11,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651894 2024-12-10T11:04:11,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651894) roll requested 2024-12-10T11:04:11,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651900 2024-12-10T11:04:11,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651900 2024-12-10T11:04:11,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651900) roll requested 2024-12-10T11:04:11,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651907 2024-12-10T11:04:11,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651907 2024-12-10T11:04:11,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651907) roll requested 2024-12-10T11:04:11,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651913 2024-12-10T11:04:11,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651913 2024-12-10T11:04:11,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651913) roll requested 2024-12-10T11:04:11,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651920 2024-12-10T11:04:11,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651920 2024-12-10T11:04:11,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651920) roll requested 2024-12-10T11:04:11,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651926 2024-12-10T11:04:11,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651926 2024-12-10T11:04:11,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651926) roll requested 2024-12-10T11:04:11,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651933 2024-12-10T11:04:11,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651933 2024-12-10T11:04:11,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651933) roll requested 2024-12-10T11:04:11,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651940 2024-12-10T11:04:11,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651940 2024-12-10T11:04:11,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651940) roll requested 2024-12-10T11:04:11,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651947 2024-12-10T11:04:11,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651947 2024-12-10T11:04:11,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651947) roll requested 2024-12-10T11:04:11,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651955 2024-12-10T11:04:11,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651955 2024-12-10T11:04:11,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651955) roll requested 2024-12-10T11:04:11,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651963 2024-12-10T11:04:11,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651963 2024-12-10T11:04:11,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651963) roll requested 2024-12-10T11:04:11,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651971 2024-12-10T11:04:11,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651963 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651971 2024-12-10T11:04:11,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651971) roll requested 2024-12-10T11:04:11,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651981 2024-12-10T11:04:11,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651981 2024-12-10T11:04:11,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651981) roll requested 2024-12-10T11:04:11,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651990 2024-12-10T11:04:11,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:11,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651990 2024-12-10T11:04:11,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:11,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:11,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:11,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:11,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651990) roll requested 2024-12-10T11:04:11,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828651998 2024-12-10T11:04:12,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651998 2024-12-10T11:04:12,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828651998) roll requested 2024-12-10T11:04:12,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652013 2024-12-10T11:04:12,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828651998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652013 2024-12-10T11:04:12,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652013) roll requested 2024-12-10T11:04:12,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652026 2024-12-10T11:04:12,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652026 2024-12-10T11:04:12,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652026) roll requested 2024-12-10T11:04:12,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652038 2024-12-10T11:04:12,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652038 2024-12-10T11:04:12,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652038) roll requested 2024-12-10T11:04:12,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652048 2024-12-10T11:04:12,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652048 2024-12-10T11:04:12,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652048) roll requested 2024-12-10T11:04:12,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652064 2024-12-10T11:04:12,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652064 2024-12-10T11:04:12,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652064) roll requested 2024-12-10T11:04:12,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652078 2024-12-10T11:04:12,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652078 2024-12-10T11:04:12,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652078) roll requested 2024-12-10T11:04:12,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652093 2024-12-10T11:04:12,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652093 2024-12-10T11:04:12,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652093) roll requested 2024-12-10T11:04:12,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652102 2024-12-10T11:04:12,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652102 2024-12-10T11:04:12,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652102) roll requested 2024-12-10T11:04:12,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652115 2024-12-10T11:04:12,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652102 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652115 2024-12-10T11:04:12,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652115) roll requested 2024-12-10T11:04:12,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652122 2024-12-10T11:04:12,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652122 2024-12-10T11:04:12,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652122) roll requested 2024-12-10T11:04:12,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652132 2024-12-10T11:04:12,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652132 2024-12-10T11:04:12,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652132) roll requested 2024-12-10T11:04:12,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652140 2024-12-10T11:04:12,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652140 2024-12-10T11:04:12,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652140) roll requested 2024-12-10T11:04:12,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652148 2024-12-10T11:04:12,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652148 2024-12-10T11:04:12,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652148) roll requested 2024-12-10T11:04:12,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652156 2024-12-10T11:04:12,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652156 2024-12-10T11:04:12,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652156) roll requested 2024-12-10T11:04:12,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652163 2024-12-10T11:04:12,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652163 2024-12-10T11:04:12,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652163) roll requested 2024-12-10T11:04:12,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652171 2024-12-10T11:04:12,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652171 2024-12-10T11:04:12,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652171) roll requested 2024-12-10T11:04:12,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652179 2024-12-10T11:04:12,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652179 2024-12-10T11:04:12,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652179) roll requested 2024-12-10T11:04:12,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652190 2024-12-10T11:04:12,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652190 2024-12-10T11:04:12,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652190) roll requested 2024-12-10T11:04:12,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652197 2024-12-10T11:04:12,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652197 2024-12-10T11:04:12,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652197) roll requested 2024-12-10T11:04:12,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652204 2024-12-10T11:04:12,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652204 2024-12-10T11:04:12,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652204) roll requested 2024-12-10T11:04:12,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652212 2024-12-10T11:04:12,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652212 2024-12-10T11:04:12,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652212) roll requested 2024-12-10T11:04:12,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652219 2024-12-10T11:04:12,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652219 2024-12-10T11:04:12,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652219) roll requested 2024-12-10T11:04:12,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652226 2024-12-10T11:04:12,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652226 2024-12-10T11:04:12,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652226) roll requested 2024-12-10T11:04:12,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652234 2024-12-10T11:04:12,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652234 2024-12-10T11:04:12,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652234) roll requested 2024-12-10T11:04:12,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652241 2024-12-10T11:04:12,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652241 2024-12-10T11:04:12,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652241) roll requested 2024-12-10T11:04:12,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652257 2024-12-10T11:04:12,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652257 2024-12-10T11:04:12,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652257) roll requested 2024-12-10T11:04:12,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652273 2024-12-10T11:04:12,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652273 2024-12-10T11:04:12,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652273) roll requested 2024-12-10T11:04:12,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652292 2024-12-10T11:04:12,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652292 2024-12-10T11:04:12,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652292) roll requested 2024-12-10T11:04:12,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652305 2024-12-10T11:04:12,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652305 2024-12-10T11:04:12,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652305) roll requested 2024-12-10T11:04:12,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652312 2024-12-10T11:04:12,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652312 2024-12-10T11:04:12,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652312) roll requested 2024-12-10T11:04:12,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652321 2024-12-10T11:04:12,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652321 2024-12-10T11:04:12,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652321) roll requested 2024-12-10T11:04:12,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652331 2024-12-10T11:04:12,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652331 2024-12-10T11:04:12,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652331) roll requested 2024-12-10T11:04:12,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652339 2024-12-10T11:04:12,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652339 2024-12-10T11:04:12,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652339) roll requested 2024-12-10T11:04:12,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652349 2024-12-10T11:04:12,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652349 2024-12-10T11:04:12,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652349) roll requested 2024-12-10T11:04:12,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652357 2024-12-10T11:04:12,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652357 2024-12-10T11:04:12,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652357) roll requested 2024-12-10T11:04:12,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652366 2024-12-10T11:04:12,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652366 2024-12-10T11:04:12,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652366) roll requested 2024-12-10T11:04:12,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652374 2024-12-10T11:04:12,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652374 2024-12-10T11:04:12,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652374) roll requested 2024-12-10T11:04:12,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652381 2024-12-10T11:04:12,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652381 2024-12-10T11:04:12,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652381) roll requested 2024-12-10T11:04:12,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652389 2024-12-10T11:04:12,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652389 2024-12-10T11:04:12,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652389) roll requested 2024-12-10T11:04:12,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652395 2024-12-10T11:04:12,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652395 2024-12-10T11:04:12,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652395) roll requested 2024-12-10T11:04:12,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652402 2024-12-10T11:04:12,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652402 2024-12-10T11:04:12,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652402) roll requested 2024-12-10T11:04:12,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652409 2024-12-10T11:04:12,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652409 2024-12-10T11:04:12,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652409) roll requested 2024-12-10T11:04:12,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652416 2024-12-10T11:04:12,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652416 2024-12-10T11:04:12,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652416) roll requested 2024-12-10T11:04:12,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652423 2024-12-10T11:04:12,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652423 2024-12-10T11:04:12,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652423) roll requested 2024-12-10T11:04:12,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652430 2024-12-10T11:04:12,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652430 2024-12-10T11:04:12,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652430) roll requested 2024-12-10T11:04:12,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652436 2024-12-10T11:04:12,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652436 2024-12-10T11:04:12,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652436) roll requested 2024-12-10T11:04:12,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652443 2024-12-10T11:04:12,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652443 2024-12-10T11:04:12,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652443) roll requested 2024-12-10T11:04:12,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652450 2024-12-10T11:04:12,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652450 2024-12-10T11:04:12,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652450) roll requested 2024-12-10T11:04:12,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652456 2024-12-10T11:04:12,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652456 2024-12-10T11:04:12,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652456) roll requested 2024-12-10T11:04:12,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652464 2024-12-10T11:04:12,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652464 2024-12-10T11:04:12,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652464) roll requested 2024-12-10T11:04:12,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652470 2024-12-10T11:04:12,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652470 2024-12-10T11:04:12,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652470) roll requested 2024-12-10T11:04:12,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652477 2024-12-10T11:04:12,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652477 2024-12-10T11:04:12,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652477) roll requested 2024-12-10T11:04:12,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652484 2024-12-10T11:04:12,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652484 2024-12-10T11:04:12,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652484) roll requested 2024-12-10T11:04:12,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652495 2024-12-10T11:04:12,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652495 2024-12-10T11:04:12,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652495) roll requested 2024-12-10T11:04:12,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652505 2024-12-10T11:04:12,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652505 2024-12-10T11:04:12,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652505) roll requested 2024-12-10T11:04:12,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652512 2024-12-10T11:04:12,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652512 2024-12-10T11:04:12,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652512) roll requested 2024-12-10T11:04:12,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652520 2024-12-10T11:04:12,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652520 2024-12-10T11:04:12,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652520) roll requested 2024-12-10T11:04:12,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652527 2024-12-10T11:04:12,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652527 2024-12-10T11:04:12,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652527) roll requested 2024-12-10T11:04:12,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652534 2024-12-10T11:04:12,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652534 2024-12-10T11:04:12,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652534) roll requested 2024-12-10T11:04:12,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652541 2024-12-10T11:04:12,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652541 2024-12-10T11:04:12,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652541) roll requested 2024-12-10T11:04:12,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652548 2024-12-10T11:04:12,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652548 2024-12-10T11:04:12,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652548) roll requested 2024-12-10T11:04:12,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652555 2024-12-10T11:04:12,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652555 2024-12-10T11:04:12,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652555) roll requested 2024-12-10T11:04:12,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652562 2024-12-10T11:04:12,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652562 2024-12-10T11:04:12,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652562) roll requested 2024-12-10T11:04:12,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652569 2024-12-10T11:04:12,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652569 2024-12-10T11:04:12,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652569) roll requested 2024-12-10T11:04:12,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652576 2024-12-10T11:04:12,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652576 2024-12-10T11:04:12,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652576) roll requested 2024-12-10T11:04:12,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652582 2024-12-10T11:04:12,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652582 2024-12-10T11:04:12,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652582) roll requested 2024-12-10T11:04:12,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652589 2024-12-10T11:04:12,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652589 2024-12-10T11:04:12,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652589) roll requested 2024-12-10T11:04:12,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652595 2024-12-10T11:04:12,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652595 2024-12-10T11:04:12,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652595) roll requested 2024-12-10T11:04:12,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652602 2024-12-10T11:04:12,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652602 2024-12-10T11:04:12,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652602) roll requested 2024-12-10T11:04:12,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652608 2024-12-10T11:04:12,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652608 2024-12-10T11:04:12,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652608) roll requested 2024-12-10T11:04:12,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652618 2024-12-10T11:04:12,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652618 2024-12-10T11:04:12,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652618) roll requested 2024-12-10T11:04:12,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652624 2024-12-10T11:04:12,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652624 2024-12-10T11:04:12,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652624) roll requested 2024-12-10T11:04:12,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652631 2024-12-10T11:04:12,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652631 2024-12-10T11:04:12,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652631) roll requested 2024-12-10T11:04:12,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652638 2024-12-10T11:04:12,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652638 2024-12-10T11:04:12,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652638) roll requested 2024-12-10T11:04:12,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652644 2024-12-10T11:04:12,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652644 2024-12-10T11:04:12,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652644) roll requested 2024-12-10T11:04:12,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652651 2024-12-10T11:04:12,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652651 2024-12-10T11:04:12,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652651) roll requested 2024-12-10T11:04:12,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652658 2024-12-10T11:04:12,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652658 2024-12-10T11:04:12,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652658) roll requested 2024-12-10T11:04:12,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652664 2024-12-10T11:04:12,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652664 2024-12-10T11:04:12,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652664) roll requested 2024-12-10T11:04:12,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652671 2024-12-10T11:04:12,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652671 2024-12-10T11:04:12,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652671) roll requested 2024-12-10T11:04:12,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652678 2024-12-10T11:04:12,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652678 2024-12-10T11:04:12,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652678) roll requested 2024-12-10T11:04:12,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652684 2024-12-10T11:04:12,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652684 2024-12-10T11:04:12,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652684) roll requested 2024-12-10T11:04:12,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652691 2024-12-10T11:04:12,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652691 2024-12-10T11:04:12,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,698 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,698 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652691) roll requested 2024-12-10T11:04:12,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652698 2024-12-10T11:04:12,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652698 2024-12-10T11:04:12,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652698) roll requested 2024-12-10T11:04:12,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652705 2024-12-10T11:04:12,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652705 2024-12-10T11:04:12,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,711 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652705) roll requested 2024-12-10T11:04:12,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652712 2024-12-10T11:04:12,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652712 2024-12-10T11:04:12,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652712) roll requested 2024-12-10T11:04:12,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652718 2024-12-10T11:04:12,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652718 2024-12-10T11:04:12,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652718) roll requested 2024-12-10T11:04:12,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652725 2024-12-10T11:04:12,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652725 2024-12-10T11:04:12,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652725) roll requested 2024-12-10T11:04:12,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652733 2024-12-10T11:04:12,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652733 2024-12-10T11:04:12,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652733) roll requested 2024-12-10T11:04:12,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652740 2024-12-10T11:04:12,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652740 2024-12-10T11:04:12,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652740) roll requested 2024-12-10T11:04:12,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652747 2024-12-10T11:04:12,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652747 2024-12-10T11:04:12,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652747) roll requested 2024-12-10T11:04:12,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652753 2024-12-10T11:04:12,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652753 2024-12-10T11:04:12,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652753) roll requested 2024-12-10T11:04:12,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652760 2024-12-10T11:04:12,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652760 2024-12-10T11:04:12,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652760) roll requested 2024-12-10T11:04:12,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652766 2024-12-10T11:04:12,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652766 2024-12-10T11:04:12,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652766) roll requested 2024-12-10T11:04:12,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652774 2024-12-10T11:04:12,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652774 2024-12-10T11:04:12,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652774) roll requested 2024-12-10T11:04:12,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652780 2024-12-10T11:04:12,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652780 2024-12-10T11:04:12,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652780) roll requested 2024-12-10T11:04:12,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652787 2024-12-10T11:04:12,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652787 2024-12-10T11:04:12,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652787) roll requested 2024-12-10T11:04:12,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652793 2024-12-10T11:04:12,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652793 2024-12-10T11:04:12,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652793) roll requested 2024-12-10T11:04:12,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652800 2024-12-10T11:04:12,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652800 2024-12-10T11:04:12,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652800) roll requested 2024-12-10T11:04:12,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652806 2024-12-10T11:04:12,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652806 2024-12-10T11:04:12,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652806) roll requested 2024-12-10T11:04:12,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652813 2024-12-10T11:04:12,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652813 2024-12-10T11:04:12,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652813) roll requested 2024-12-10T11:04:12,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652819 2024-12-10T11:04:12,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652819 2024-12-10T11:04:12,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,826 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,826 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652819) roll requested 2024-12-10T11:04:12,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652826 2024-12-10T11:04:12,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652826 2024-12-10T11:04:12,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652826) roll requested 2024-12-10T11:04:12,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652833 2024-12-10T11:04:12,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652826 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652833 2024-12-10T11:04:12,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652833) roll requested 2024-12-10T11:04:12,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652839 2024-12-10T11:04:12,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652839 2024-12-10T11:04:12,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652839) roll requested 2024-12-10T11:04:12,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652846 2024-12-10T11:04:12,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652846 2024-12-10T11:04:12,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652846) roll requested 2024-12-10T11:04:12,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652852 2024-12-10T11:04:12,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652852 2024-12-10T11:04:12,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652852) roll requested 2024-12-10T11:04:12,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652858 2024-12-10T11:04:12,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652858 2024-12-10T11:04:12,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652858) roll requested 2024-12-10T11:04:12,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652865 2024-12-10T11:04:12,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652865 2024-12-10T11:04:12,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652865) roll requested 2024-12-10T11:04:12,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652872 2024-12-10T11:04:12,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652872 2024-12-10T11:04:12,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652872) roll requested 2024-12-10T11:04:12,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652879 2024-12-10T11:04:12,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652879 2024-12-10T11:04:12,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652879) roll requested 2024-12-10T11:04:12,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652886 2024-12-10T11:04:12,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652886 2024-12-10T11:04:12,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652886) roll requested 2024-12-10T11:04:12,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652892 2024-12-10T11:04:12,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652892 2024-12-10T11:04:12,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652892) roll requested 2024-12-10T11:04:12,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652899 2024-12-10T11:04:12,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652899 2024-12-10T11:04:12,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652899) roll requested 2024-12-10T11:04:12,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652907 2024-12-10T11:04:12,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652907 2024-12-10T11:04:12,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652907) roll requested 2024-12-10T11:04:12,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652913 2024-12-10T11:04:12,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652913 2024-12-10T11:04:12,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652913) roll requested 2024-12-10T11:04:12,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652920 2024-12-10T11:04:12,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652920 2024-12-10T11:04:12,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652920) roll requested 2024-12-10T11:04:12,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652926 2024-12-10T11:04:12,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652926 2024-12-10T11:04:12,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652926) roll requested 2024-12-10T11:04:12,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652932 2024-12-10T11:04:12,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652932 2024-12-10T11:04:12,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652932) roll requested 2024-12-10T11:04:12,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652939 2024-12-10T11:04:12,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652939 2024-12-10T11:04:12,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652939) roll requested 2024-12-10T11:04:12,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652945 2024-12-10T11:04:12,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652945 2024-12-10T11:04:12,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652945) roll requested 2024-12-10T11:04:12,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652952 2024-12-10T11:04:12,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652952 2024-12-10T11:04:12,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652952) roll requested 2024-12-10T11:04:12,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652959 2024-12-10T11:04:12,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652959 2024-12-10T11:04:12,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652959) roll requested 2024-12-10T11:04:12,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652965 2024-12-10T11:04:12,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652965 2024-12-10T11:04:12,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652965) roll requested 2024-12-10T11:04:12,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652972 2024-12-10T11:04:12,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652972 2024-12-10T11:04:12,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652972) roll requested 2024-12-10T11:04:12,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652979 2024-12-10T11:04:12,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652979 2024-12-10T11:04:12,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652979) roll requested 2024-12-10T11:04:12,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652985 2024-12-10T11:04:12,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652985 2024-12-10T11:04:12,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652985) roll requested 2024-12-10T11:04:12,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652991 2024-12-10T11:04:12,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:12,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652991 2024-12-10T11:04:12,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:12,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:12,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:12,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:12,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652991) roll requested 2024-12-10T11:04:12,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828652998 2024-12-10T11:04:13,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652998 2024-12-10T11:04:13,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828652998) roll requested 2024-12-10T11:04:13,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653005 2024-12-10T11:04:13,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828652998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653005 2024-12-10T11:04:13,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653005) roll requested 2024-12-10T11:04:13,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653011 2024-12-10T11:04:13,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653011 2024-12-10T11:04:13,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653011) roll requested 2024-12-10T11:04:13,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653018 2024-12-10T11:04:13,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653018 2024-12-10T11:04:13,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653018) roll requested 2024-12-10T11:04:13,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653024 2024-12-10T11:04:13,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653024 2024-12-10T11:04:13,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653024) roll requested 2024-12-10T11:04:13,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653031 2024-12-10T11:04:13,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653031 2024-12-10T11:04:13,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653031) roll requested 2024-12-10T11:04:13,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653037 2024-12-10T11:04:13,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653037 2024-12-10T11:04:13,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653037) roll requested 2024-12-10T11:04:13,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653044 2024-12-10T11:04:13,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653044 2024-12-10T11:04:13,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653044) roll requested 2024-12-10T11:04:13,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653051 2024-12-10T11:04:13,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653051 2024-12-10T11:04:13,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653051) roll requested 2024-12-10T11:04:13,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653058 2024-12-10T11:04:13,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653058 2024-12-10T11:04:13,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653058) roll requested 2024-12-10T11:04:13,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653065 2024-12-10T11:04:13,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653065 2024-12-10T11:04:13,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653065) roll requested 2024-12-10T11:04:13,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653072 2024-12-10T11:04:13,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653072 2024-12-10T11:04:13,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653072) roll requested 2024-12-10T11:04:13,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653078 2024-12-10T11:04:13,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653078 2024-12-10T11:04:13,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653078) roll requested 2024-12-10T11:04:13,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653085 2024-12-10T11:04:13,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653085 2024-12-10T11:04:13,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653085) roll requested 2024-12-10T11:04:13,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653092 2024-12-10T11:04:13,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653092 2024-12-10T11:04:13,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653092) roll requested 2024-12-10T11:04:13,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653098 2024-12-10T11:04:13,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653098 2024-12-10T11:04:13,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1206, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653098) roll requested 2024-12-10T11:04:13,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653105 2024-12-10T11:04:13,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653105 2024-12-10T11:04:13,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1207, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653105) roll requested 2024-12-10T11:04:13,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653114 2024-12-10T11:04:13,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653114 2024-12-10T11:04:13,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1208, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653114) roll requested 2024-12-10T11:04:13,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653121 2024-12-10T11:04:13,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653121 2024-12-10T11:04:13,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1209, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653121) roll requested 2024-12-10T11:04:13,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653130 2024-12-10T11:04:13,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653130 2024-12-10T11:04:13,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1210, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653130) roll requested 2024-12-10T11:04:13,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653136 2024-12-10T11:04:13,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653136 2024-12-10T11:04:13,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1211, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653136) roll requested 2024-12-10T11:04:13,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653143 2024-12-10T11:04:13,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653143 2024-12-10T11:04:13,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1212, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653143) roll requested 2024-12-10T11:04:13,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653150 2024-12-10T11:04:13,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653150 2024-12-10T11:04:13,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1213, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653150) roll requested 2024-12-10T11:04:13,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653157 2024-12-10T11:04:13,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653157 2024-12-10T11:04:13,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1214, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653157) roll requested 2024-12-10T11:04:13,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653163 2024-12-10T11:04:13,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653163 2024-12-10T11:04:13,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1215, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653163) roll requested 2024-12-10T11:04:13,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653170 2024-12-10T11:04:13,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653170 2024-12-10T11:04:13,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1216, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653170) roll requested 2024-12-10T11:04:13,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653177 2024-12-10T11:04:13,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653177 2024-12-10T11:04:13,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1217, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653177) roll requested 2024-12-10T11:04:13,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653183 2024-12-10T11:04:13,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653183 2024-12-10T11:04:13,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1218, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653183) roll requested 2024-12-10T11:04:13,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653190 2024-12-10T11:04:13,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653190 2024-12-10T11:04:13,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1219, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653190) roll requested 2024-12-10T11:04:13,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653197 2024-12-10T11:04:13,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653197 2024-12-10T11:04:13,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1220, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653197) roll requested 2024-12-10T11:04:13,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653204 2024-12-10T11:04:13,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653204 2024-12-10T11:04:13,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1221, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653204) roll requested 2024-12-10T11:04:13,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653210 2024-12-10T11:04:13,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653210 2024-12-10T11:04:13,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1222, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653210) roll requested 2024-12-10T11:04:13,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653217 2024-12-10T11:04:13,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653217 2024-12-10T11:04:13,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1223, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653217) roll requested 2024-12-10T11:04:13,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653223 2024-12-10T11:04:13,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653223 2024-12-10T11:04:13,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1224, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653223) roll requested 2024-12-10T11:04:13,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653229 2024-12-10T11:04:13,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653229 2024-12-10T11:04:13,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1225, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653229) roll requested 2024-12-10T11:04:13,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653236 2024-12-10T11:04:13,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653236 2024-12-10T11:04:13,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1226, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653236) roll requested 2024-12-10T11:04:13,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653243 2024-12-10T11:04:13,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653243 2024-12-10T11:04:13,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1227, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653243) roll requested 2024-12-10T11:04:13,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653250 2024-12-10T11:04:13,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653250 2024-12-10T11:04:13,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1228, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653250) roll requested 2024-12-10T11:04:13,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653257 2024-12-10T11:04:13,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653257 2024-12-10T11:04:13,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1229, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653257) roll requested 2024-12-10T11:04:13,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653263 2024-12-10T11:04:13,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653263 2024-12-10T11:04:13,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1230, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653263) roll requested 2024-12-10T11:04:13,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653270 2024-12-10T11:04:13,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653270 2024-12-10T11:04:13,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1231, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653270) roll requested 2024-12-10T11:04:13,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653277 2024-12-10T11:04:13,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653277 2024-12-10T11:04:13,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1232, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653277) roll requested 2024-12-10T11:04:13,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653283 2024-12-10T11:04:13,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653283 2024-12-10T11:04:13,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1233, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653283) roll requested 2024-12-10T11:04:13,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653290 2024-12-10T11:04:13,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653290 2024-12-10T11:04:13,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1234, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653290) roll requested 2024-12-10T11:04:13,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653297 2024-12-10T11:04:13,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653297 2024-12-10T11:04:13,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1235, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653297) roll requested 2024-12-10T11:04:13,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653306 2024-12-10T11:04:13,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653306 2024-12-10T11:04:13,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1236, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653306) roll requested 2024-12-10T11:04:13,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653312 2024-12-10T11:04:13,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653312 2024-12-10T11:04:13,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1237, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653312) roll requested 2024-12-10T11:04:13,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653319 2024-12-10T11:04:13,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653319 2024-12-10T11:04:13,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1238, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653319) roll requested 2024-12-10T11:04:13,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653325 2024-12-10T11:04:13,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653325 2024-12-10T11:04:13,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1239, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653325) roll requested 2024-12-10T11:04:13,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653332 2024-12-10T11:04:13,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653332 2024-12-10T11:04:13,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1240, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653332) roll requested 2024-12-10T11:04:13,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653338 2024-12-10T11:04:13,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653338 2024-12-10T11:04:13,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1241, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653338) roll requested 2024-12-10T11:04:13,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653345 2024-12-10T11:04:13,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653345 2024-12-10T11:04:13,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1242, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653345) roll requested 2024-12-10T11:04:13,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653351 2024-12-10T11:04:13,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653351 2024-12-10T11:04:13,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1243, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653351) roll requested 2024-12-10T11:04:13,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653358 2024-12-10T11:04:13,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653358 2024-12-10T11:04:13,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,365 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1244, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653358) roll requested 2024-12-10T11:04:13,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653365 2024-12-10T11:04:13,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653365 2024-12-10T11:04:13,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1245, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653365) roll requested 2024-12-10T11:04:13,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653371 2024-12-10T11:04:13,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653371 2024-12-10T11:04:13,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1246, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653371) roll requested 2024-12-10T11:04:13,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653378 2024-12-10T11:04:13,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653378 2024-12-10T11:04:13,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1247, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653378) roll requested 2024-12-10T11:04:13,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653385 2024-12-10T11:04:13,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653385 2024-12-10T11:04:13,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1248, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653385) roll requested 2024-12-10T11:04:13,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653392 2024-12-10T11:04:13,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653392 2024-12-10T11:04:13,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1249, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653392) roll requested 2024-12-10T11:04:13,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653398 2024-12-10T11:04:13,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653398 2024-12-10T11:04:13,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1250, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653398) roll requested 2024-12-10T11:04:13,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653406 2024-12-10T11:04:13,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653406 2024-12-10T11:04:13,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1251, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653406) roll requested 2024-12-10T11:04:13,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653413 2024-12-10T11:04:13,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653413 2024-12-10T11:04:13,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1252, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653413) roll requested 2024-12-10T11:04:13,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653419 2024-12-10T11:04:13,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653419 2024-12-10T11:04:13,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1253, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653419) roll requested 2024-12-10T11:04:13,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653426 2024-12-10T11:04:13,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653426 2024-12-10T11:04:13,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1254, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653426) roll requested 2024-12-10T11:04:13,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653432 2024-12-10T11:04:13,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653432 2024-12-10T11:04:13,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1255, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653432) roll requested 2024-12-10T11:04:13,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653439 2024-12-10T11:04:13,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653439 2024-12-10T11:04:13,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1256, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653439) roll requested 2024-12-10T11:04:13,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653446 2024-12-10T11:04:13,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653446 2024-12-10T11:04:13,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1257, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653446) roll requested 2024-12-10T11:04:13,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653452 2024-12-10T11:04:13,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653452 2024-12-10T11:04:13,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1258, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653452) roll requested 2024-12-10T11:04:13,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653459 2024-12-10T11:04:13,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653459 2024-12-10T11:04:13,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1259, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653459) roll requested 2024-12-10T11:04:13,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653466 2024-12-10T11:04:13,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653466 2024-12-10T11:04:13,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1260, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653466) roll requested 2024-12-10T11:04:13,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653472 2024-12-10T11:04:13,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653472 2024-12-10T11:04:13,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1261, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653472) roll requested 2024-12-10T11:04:13,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653479 2024-12-10T11:04:13,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653479 2024-12-10T11:04:13,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1262, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653479) roll requested 2024-12-10T11:04:13,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653486 2024-12-10T11:04:13,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653486 2024-12-10T11:04:13,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1263, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653486) roll requested 2024-12-10T11:04:13,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653493 2024-12-10T11:04:13,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653493 2024-12-10T11:04:13,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1264, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653493) roll requested 2024-12-10T11:04:13,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653499 2024-12-10T11:04:13,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653499 2024-12-10T11:04:13,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1265, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653499) roll requested 2024-12-10T11:04:13,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653506 2024-12-10T11:04:13,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653506 2024-12-10T11:04:13,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1266, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653506) roll requested 2024-12-10T11:04:13,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653512 2024-12-10T11:04:13,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653512 2024-12-10T11:04:13,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1267, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653512) roll requested 2024-12-10T11:04:13,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653519 2024-12-10T11:04:13,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653519 2024-12-10T11:04:13,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1268, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653519) roll requested 2024-12-10T11:04:13,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653525 2024-12-10T11:04:13,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653525 2024-12-10T11:04:13,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1269, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653525) roll requested 2024-12-10T11:04:13,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653532 2024-12-10T11:04:13,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653532 2024-12-10T11:04:13,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1270, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653532) roll requested 2024-12-10T11:04:13,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653538 2024-12-10T11:04:13,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653538 2024-12-10T11:04:13,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,545 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1271, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653538) roll requested 2024-12-10T11:04:13,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653545 2024-12-10T11:04:13,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653545 2024-12-10T11:04:13,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1272, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653545) roll requested 2024-12-10T11:04:13,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653552 2024-12-10T11:04:13,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653552 2024-12-10T11:04:13,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1273, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653552) roll requested 2024-12-10T11:04:13,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653559 2024-12-10T11:04:13,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653559 2024-12-10T11:04:13,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1274, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653559) roll requested 2024-12-10T11:04:13,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653565 2024-12-10T11:04:13,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653565 2024-12-10T11:04:13,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1275, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653565) roll requested 2024-12-10T11:04:13,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653572 2024-12-10T11:04:13,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653572 2024-12-10T11:04:13,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1276, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653572) roll requested 2024-12-10T11:04:13,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653578 2024-12-10T11:04:13,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653578 2024-12-10T11:04:13,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1277, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653578) roll requested 2024-12-10T11:04:13,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653585 2024-12-10T11:04:13,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653585 2024-12-10T11:04:13,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1278, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653585) roll requested 2024-12-10T11:04:13,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653593 2024-12-10T11:04:13,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653593 2024-12-10T11:04:13,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1279, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653593) roll requested 2024-12-10T11:04:13,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653599 2024-12-10T11:04:13,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653599 2024-12-10T11:04:13,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,606 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1280, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,606 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653599) roll requested 2024-12-10T11:04:13,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653606 2024-12-10T11:04:13,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653606 2024-12-10T11:04:13,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1281, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653606) roll requested 2024-12-10T11:04:13,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653612 2024-12-10T11:04:13,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653612 2024-12-10T11:04:13,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1282, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653612) roll requested 2024-12-10T11:04:13,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653619 2024-12-10T11:04:13,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653619 2024-12-10T11:04:13,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1283, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653619) roll requested 2024-12-10T11:04:13,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653626 2024-12-10T11:04:13,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653626 2024-12-10T11:04:13,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1284, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653626) roll requested 2024-12-10T11:04:13,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653632 2024-12-10T11:04:13,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653632 2024-12-10T11:04:13,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1285, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653632) roll requested 2024-12-10T11:04:13,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653639 2024-12-10T11:04:13,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653639 2024-12-10T11:04:13,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1286, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653639) roll requested 2024-12-10T11:04:13,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653646 2024-12-10T11:04:13,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653646 2024-12-10T11:04:13,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1287, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653646) roll requested 2024-12-10T11:04:13,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653652 2024-12-10T11:04:13,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653652 2024-12-10T11:04:13,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1288, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653652) roll requested 2024-12-10T11:04:13,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653659 2024-12-10T11:04:13,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653659 2024-12-10T11:04:13,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-10T11:04:13,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1289, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733828653659) roll requested 2024-12-10T11:04:13,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733828653666 2024-12-10T11:04:13,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733828653666 2024-12-10T11:04:13,674 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T11:04:13,676 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0dbc67421fdaf4442fb699251a0dc6f4, disabling compactions & flushes 2024-12-10T11:04:13,676 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1733828623341.0dbc67421fdaf4442fb699251a0dc6f4. 2024-12-10T11:04:13,676 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1733828623341.0dbc67421fdaf4442fb699251a0dc6f4. 2024-12-10T11:04:13,677 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1733828623341.0dbc67421fdaf4442fb699251a0dc6f4. after waiting 0 ms 2024-12-10T11:04:13,677 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1733828623341.0dbc67421fdaf4442fb699251a0dc6f4. 2024-12-10T11:04:13,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-10T11:04:13,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1290, max=55; forcing (partial) flush of 1 region(s): 0dbc67421fdaf4442fb699251a0dc6f4[MyCF] 2024-12-10T11:04:13,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 0dbc67421fdaf4442fb699251a0dc6f4, because it is not online on us 2024-12-10T11:04:13,678 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T11:04:13,678 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1733828623341.0dbc67421fdaf4442fb699251a0dc6f4. 2024-12-10T11:04:13,678 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1733828623341.0dbc67421fdaf4442fb699251a0dc6f4. 2024-12-10T11:04:13,678 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0dbc67421fdaf4442fb699251a0dc6f4: Waiting for close lock at 1733828653676Disabling compacts and flushes for region at 1733828653676Disabling writes for close at 1733828653677 (+1 ms)Writing region close event to WAL at 1733828653678 (+1 ms)Closed at 1733828653678 2024-12-10T11:04:13,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:13,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T11:04:14,068 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1733828653666) 2024-12-10T11:04:14,068 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cf1dcedd-712b-3227-a3bf-467a40ce11f9 2024-12-10T11:04:14,690 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=209 (was 195) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4309 (was 5391)